diff --git a/include/wsrep/client_context.hpp b/include/wsrep/client_context.hpp index e0ca013..c757adc 100644 --- a/include/wsrep/client_context.hpp +++ b/include/wsrep/client_context.hpp @@ -470,7 +470,7 @@ namespace wsrep * only for critical conditions which would sacrifice data * consistency. */ - virtual void abort() const = 0; + virtual void abort() = 0; public: /*! @@ -485,7 +485,7 @@ namespace wsrep /*! * Enter debug synchronization point. */ - virtual void debug_sync(const char*) = 0; + virtual void debug_sync(wsrep::unique_lock&, const char*) = 0; /*! * diff --git a/include/wsrep/logger.hpp b/include/wsrep/logger.hpp index 16ddda9..f62d712 100644 --- a/include/wsrep/logger.hpp +++ b/include/wsrep/logger.hpp @@ -38,6 +38,13 @@ namespace wsrep }; + class log_warning : public log + { + public: + log_warning() + : log("WARNING") { } + }; + class log_debug : public log { public: diff --git a/src/dbms_simulator.cpp b/src/dbms_simulator.cpp index b621ab0..592b365 100644 --- a/src/dbms_simulator.cpp +++ b/src/dbms_simulator.cpp @@ -385,14 +385,14 @@ private: return 0; } bool killed() const override { return false; } - void abort() const override { ::abort(); } + void abort() override { ::abort(); } public: void store_globals() override { wsrep::client_context::store_globals(); } private: - void debug_sync(const char*) override { } + void debug_sync(wsrep::unique_lock&, const char*) override { } void debug_suicide(const char*) override { } void on_error(enum wsrep::client_error) override { } diff --git a/src/mock_client_context.hpp b/src/mock_client_context.hpp index 868a955..39c5462 100644 --- a/src/mock_client_context.hpp +++ b/src/mock_client_context.hpp @@ -31,6 +31,9 @@ namespace wsrep , bf_abort_during_wait_() , error_during_prepare_data_() , killed_before_certify_() + , sync_point_action_() + , replays_() + , aborts_() { } ~mock_client_context() { @@ -51,6 +54,7 @@ namespace wsrep tc.state(lock, wsrep::transaction_context::s_committing); tc.state(lock, wsrep::transaction_context::s_ordered_commit); tc.state(lock, wsrep::transaction_context::s_committed); + ++replays_; return 0; } void wait_for_replayers(wsrep::unique_lock& lock) @@ -76,15 +80,26 @@ namespace wsrep } bool killed() const WSREP_OVERRIDE { return killed_before_certify_; } - void abort() const WSREP_OVERRIDE { } + void abort() WSREP_OVERRIDE { ++aborts_; } void store_globals() WSREP_OVERRIDE { } - void debug_sync(const char*) WSREP_OVERRIDE { } + void debug_sync(wsrep::unique_lock& lock, + const char* sync_point) WSREP_OVERRIDE + { + lock.unlock(); + if (sync_point_action_ == sync_point) + { + wsrep_test::bf_abort_ordered(*this); + } + lock.lock(); + } void debug_suicide(const char*) WSREP_OVERRIDE { ::abort(); } void on_error(enum wsrep::client_error) { } + size_t replays() const { return replays_; } + size_t aborts() const { return aborts_; } // private: wsrep::default_mutex mutex_; @@ -95,6 +110,10 @@ namespace wsrep bool bf_abort_during_wait_; bool error_during_prepare_data_; bool killed_before_certify_; + std::string sync_point_action_; + private: + size_t replays_; + size_t aborts_; }; } diff --git a/src/mock_provider.hpp b/src/mock_provider.hpp index 8cb216b..6b314de 100644 --- a/src/mock_provider.hpp +++ b/src/mock_provider.hpp @@ -43,6 +43,10 @@ namespace wsrep wsrep::ws_meta& ws_meta) { assert(flags & wsrep::provider::flag::start_transaction); + if (next_error_) + { + return next_error_; + } if ((flags & wsrep::provider::flag::commit) == 0) { return wsrep::provider::error_provider_failed; diff --git a/src/test_utils.cpp b/src/test_utils.cpp index 126ba7f..c4505f8 100644 --- a/src/test_utils.cpp +++ b/src/test_utils.cpp @@ -11,10 +11,16 @@ void wsrep_test::bf_abort_unordered(wsrep::client_context& cc) { wsrep::unique_lock lock(cc.mutex()); - assert(cc.transaction().seqno().nil()); + assert(cc.transaction().ordered() == false); cc.bf_abort(lock, 1); } +void wsrep_test::bf_abort_ordered(wsrep::client_context& cc) +{ + wsrep::unique_lock lock(cc.mutex()); + assert(cc.transaction().ordered()); + cc.bf_abort(lock, 0); +} // BF abort method to abort transactions via provider void wsrep_test::bf_abort_provider(wsrep::mock_server_context& sc, const wsrep::transaction_context& tc, diff --git a/src/test_utils.hpp b/src/test_utils.hpp index 00af57d..4466990 100644 --- a/src/test_utils.hpp +++ b/src/test_utils.hpp @@ -21,6 +21,9 @@ namespace wsrep_test // Simple BF abort method to BF abort unordered transasctions void bf_abort_unordered(wsrep::client_context& cc); + // Simple BF abort method to BF abort unordered transasctions + void bf_abort_ordered(wsrep::client_context& cc); + // BF abort method to abort transactions via provider void bf_abort_provider(wsrep::mock_server_context& sc, const wsrep::transaction_context& tc, diff --git a/src/transaction_context.cpp b/src/transaction_context.cpp index 5007305..d70eb11 100644 --- a/src/transaction_context.cpp +++ b/src/transaction_context.cpp @@ -725,7 +725,7 @@ int wsrep::transaction_context::certify_commit( lock.lock(); assert(state() == s_certifying || state() == s_must_abort); - client_context_.debug_sync("wsrep_after_replication"); + client_context_.debug_sync(lock, "wsrep_after_certification"); int ret(1); switch (cert_ret) @@ -761,8 +761,8 @@ int wsrep::transaction_context::certify_commit( state(lock, s_must_abort); // The execution should never reach this point if the // transaction has not generated any keys or data. + wsrep::log_warning() << "Transaction was missing in provider"; client_context_.override_error(wsrep::e_error_during_commit); - assert(0); break; case wsrep::provider::error_bf_abort: // Transaction was replicated succesfully and it was either @@ -797,15 +797,20 @@ int wsrep::transaction_context::certify_commit( client_context_.override_error(wsrep::e_error_during_commit); break; case wsrep::provider::error_fatal: + client_context_.override_error(wsrep::e_error_during_commit); + state(lock, s_must_abort); client_context_.abort(); break; case wsrep::provider::error_not_implemented: case wsrep::provider::error_not_allowed: client_context_.override_error(wsrep::e_error_during_commit); state(lock, s_must_abort); - assert(0); + wsrep::log_warning() << "Certification operation was not allowed: " + << "id: " << id().get() + << " flags: " << std::hex << flags() << std::dec; break; default: + state(lock, s_must_abort); client_context_.override_error(wsrep::e_error_during_commit); break; } diff --git a/src/transaction_context_test.cpp b/src/transaction_context_test.cpp index 2013ad4..36330da 100644 --- a/src/transaction_context_test.cpp +++ b/src/transaction_context_test.cpp @@ -550,6 +550,334 @@ BOOST_FIXTURE_TEST_CASE_TEMPLATE( BOOST_REQUIRE(cc.current_error() == wsrep::e_success); } +// +// Test a 1PC transaction which gets "warning error" from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_context_1pc_warning_error_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_context& sc(T::sc); + wsrep::client_context& cc(T::cc); + const wsrep::transaction_context& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(1); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing); + + sc.provider().inject_error(wsrep::provider::error_warning); + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().inject_error(wsrep::provider::success); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); +} + +// +// Test a 1PC transaction which gets transaction missing from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_context_1pc_transaction_missing_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_context& sc(T::sc); + wsrep::client_context& cc(T::cc); + const wsrep::transaction_context& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(1); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing); + + sc.provider().inject_error(wsrep::provider::error_transaction_missing); + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().inject_error(wsrep::provider::success); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); +} + +// +// Test a 1PC transaction which gets size exceeded error from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_context_1pc_size_exceeded_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_context& sc(T::sc); + wsrep::client_context& cc(T::cc); + const wsrep::transaction_context& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(1); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing); + + sc.provider().inject_error(wsrep::provider::error_size_exceeded); + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().inject_error(wsrep::provider::success); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); +} + +// +// Test a 1PC transaction which gets connection failed error from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_context_1pc_connection_failed_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_context& sc(T::sc); + wsrep::client_context& cc(T::cc); + const wsrep::transaction_context& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(1); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing); + + sc.provider().inject_error(wsrep::provider::error_connection_failed); + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().inject_error(wsrep::provider::success); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); +} + +// +// Test a 1PC transaction which gets not allowed error from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_context_1pc_no_allowed_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_context& sc(T::sc); + wsrep::client_context& cc(T::cc); + const wsrep::transaction_context& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(1); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing); + + sc.provider().inject_error(wsrep::provider::error_not_allowed); + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().inject_error(wsrep::provider::success); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); +} + +// +// Test a 1PC transaction which gets fatal error from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_context_1pc_fatal_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_context& sc(T::sc); + wsrep::mock_client_context& cc(T::cc); + const wsrep::transaction_context& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(1); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing); + + sc.provider().inject_error(wsrep::provider::error_fatal); + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().inject_error(wsrep::provider::success); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); + BOOST_REQUIRE(cc.aborts() == 1); +} + +// +// Test a 1PC transaction which gets unknown from certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_context_1pc_unknown_from_certify, T, + replicating_fixtures, T) +{ + wsrep::mock_server_context& sc(T::sc); + wsrep::mock_client_context& cc(T::cc); + const wsrep::transaction_context& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(1); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing); + + sc.provider().inject_error(wsrep::provider::error_unknown); + + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_abort); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(tc.ordered() == false); + + sc.provider().inject_error(wsrep::provider::success); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborting); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_aborted); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_error_during_commit); +} + +// +// Test a 1PC transaction which gets BF aborted before grabbing lock +// after certify call +// +BOOST_FIXTURE_TEST_CASE_TEMPLATE( + transaction_context_1pc_bf_abort_after_certify_regain_lock, T, + replicating_fixtures, T) +{ + // wsrep::mock_server_context& sc(T::sc); + wsrep::mock_client_context& cc(T::cc); + const wsrep::transaction_context& tc(T::tc); + + // Start a new transaction with ID 1 + cc.start_transaction(1); + BOOST_REQUIRE(tc.active()); + BOOST_REQUIRE(tc.id() == wsrep::transaction_id(1)); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_executing); + + cc.sync_point_action_ = "wsrep_after_certification"; + // Run before commit + BOOST_REQUIRE(cc.before_commit()); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_replay); + BOOST_REQUIRE(tc.certified() == true); + BOOST_REQUIRE(tc.ordered() == true); + + // Rollback sequence + BOOST_REQUIRE(cc.before_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_replay); + BOOST_REQUIRE(cc.after_rollback() == 0); + BOOST_REQUIRE(tc.state() == wsrep::transaction_context::s_must_replay); + + // Cleanup after statement + cc.after_statement(); + BOOST_REQUIRE(cc.replays() == 1); + BOOST_REQUIRE(tc.active() == false); + BOOST_REQUIRE(tc.ordered() == false); + BOOST_REQUIRE(tc.certified() == false); + BOOST_REQUIRE(cc.current_error() == wsrep::e_success); +} + // // Test a 2PC transaction which gets BF aborted when trying to grab // commit order.