1
0
mirror of https://github.com/codership/wsrep-lib.git synced 2025-07-30 07:23:07 +03:00

Fixed several issues with BF aborts and error reporting.

Client context override_error() is now implemented in
client_context base class. On error virtual method will be called
to notify derived classes.

Started refactoring logging utility.
This commit is contained in:
Teemu Ollakka
2018-05-02 13:55:38 +03:00
parent e3ca0d699b
commit 8e501fc03b
10 changed files with 286 additions and 111 deletions

View File

@ -3,7 +3,6 @@
//
#include "client_context.hpp"
#include "transaction_context.hpp"
#include "compiler.hpp"
#include <sstream>
@ -26,15 +25,15 @@ int trrep::client_context::before_command()
* \todo Wait until the possible synchronous rollback
* has been finished.
*/
trrep::unique_lock<trrep::mutex> lock(mutex_);
while (transaction_.state() == trrep::transaction_context::s_aborting)
{
// cond_.wait(lock);
}
}
state(lock, s_exec);
if (transaction_.state() == trrep::transaction_context::s_must_abort ||
transaction_.state() == trrep::transaction_context::s_aborted)
if (transaction_.active() &&
(transaction_.state() == trrep::transaction_context::s_must_abort ||
transaction_.state() == trrep::transaction_context::s_aborted))
{
return 1;
}
@ -44,11 +43,15 @@ int trrep::client_context::before_command()
void trrep::client_context::after_command()
{
trrep::unique_lock<trrep::mutex> lock(mutex_);
if (transaction_.state() == trrep::transaction_context::s_must_abort)
if (transaction_.active() &&
transaction_.state() == trrep::transaction_context::s_must_abort)
{
override_error(trrep::e_deadlock_error);
lock.unlock();
rollback(transaction_);
transaction_.after_statement();
lock.lock();
assert(transaction_.state() == trrep::transaction_context::s_aborted);
}
state(lock, s_idle);
}
@ -68,8 +71,10 @@ int trrep::client_context::before_statement()
}
#endif // 0
if (transaction_.state() == trrep::transaction_context::s_must_abort)
if (transaction_.active() &&
transaction_.state() == trrep::transaction_context::s_must_abort)
{
override_error(trrep::e_deadlock_error);
lock.unlock();
rollback(transaction_);
lock.lock();

View File

@ -57,6 +57,17 @@ namespace trrep
e_append_fragment_error
};
static inline std::string to_string(enum client_error error)
{
switch (error)
{
case e_success: return "success";
case e_error_during_commit: return "error_during_commit";
case e_deadlock_error: return "deadlock_error";
case e_append_fragment_error: return "append_fragment_error";
}
return "unknown";
}
class client_id
{
public:
@ -285,6 +296,23 @@ namespace trrep
{
return transaction_;
}
void debug_log_level(int level) { debug_log_level_ = level; }
int debug_log_level() const
{
return std::max(debug_log_level_,
server_context_.debug_log_level());
}
void reset_error()
{
current_error_ = trrep::e_success;
}
enum trrep::client_error current_error() const
{
return current_error_;
}
protected:
/*!
* Client context constuctor. This is protected so that it
@ -301,6 +329,8 @@ namespace trrep
, state_(s_idle)
, transaction_(*this)
, allow_dirty_reads_()
, debug_log_level_(0)
, current_error_(trrep::e_success)
{ }
private:
@ -400,11 +430,6 @@ namespace trrep
return 0;
}
/*!
*
*/
virtual void override_error(const trrep::client_error&) = 0;
/*!
* Return true if the current client operation was killed.
*/
@ -432,6 +457,22 @@ namespace trrep
*/
virtual void debug_suicide(const std::string&) = 0;
/*!
* Notify the implementation about an error.
*/
virtual void on_error(enum trrep::client_error error) = 0;
/*!
*
*/
void override_error(enum trrep::client_error error)
{
if (current_error_ != trrep::e_success &&
error == trrep::e_success)
{
throw trrep::runtime_error("Overriding error with success");
}
current_error_ = error;
}
trrep::mutex& mutex_;
trrep::server_context& server_context_;
@ -444,6 +485,8 @@ namespace trrep
* semantics.
*/
bool allow_dirty_reads_;
int debug_log_level_;
trrep::client_error current_error_;
};

View File

@ -8,6 +8,7 @@
// through trrep interface.
//
#include "server_context.hpp"
#include "client_context.hpp"
#include "transaction_context.hpp"
@ -39,12 +40,14 @@ struct dbms_simulator_params
size_t n_transactions;
std::string wsrep_provider;
std::string wsrep_provider_options;
int debug_log_level;
dbms_simulator_params()
: n_servers(0)
, n_clients(0)
, n_transactions(0)
, wsrep_provider()
, wsrep_provider_options()
, debug_log_level(0)
{ }
};
@ -54,7 +57,7 @@ public:
dbms_storage_engine()
: mutex_()
, transactions_()
, alg_freq_(1)
, alg_freq_(100)
, bf_aborts_()
{ }
@ -66,6 +69,9 @@ public:
, txc_()
{
}
bool active() const { return txc_ != nullptr; }
void start(trrep::transaction_context* txc)
{
trrep::unique_lock<trrep::mutex> lock(se_.mutex_);
@ -74,16 +80,32 @@ public:
::abort();
}
txc_ = txc;
}
~transaction()
void commit()
{
if (txc_)
{
trrep::unique_lock<trrep::mutex> lock(se_.mutex_);
se_.transactions_.erase(txc_);
}
txc_ = nullptr;
}
void abort()
{
if (txc_)
{
trrep::unique_lock<trrep::mutex> lock(se_.mutex_);
se_.transactions_.erase(txc_);
}
txc_ = nullptr;
}
~transaction()
{
abort();
}
transaction(const transaction&) = delete;
@ -97,7 +119,7 @@ public:
void bf_abort_some(const trrep::transaction_context& txc)
{
trrep::unique_lock<trrep::mutex> lock(mutex_);
if ((std::rand() % alg_freq_) == 0)
if (alg_freq_ && (std::rand() % alg_freq_) == 0)
{
if (transactions_.empty() == false)
{
@ -265,6 +287,7 @@ public:
: trrep::client_context(mutex_, server, id, mode)
, mutex_()
, server_(server)
, se_trx_(server_.storage_engine())
, n_transactions_(n_transactions)
{ }
@ -302,6 +325,7 @@ private:
<< "state: "
<< trrep::to_string(transaction_context.state());
transaction_context.before_rollback();
se_trx_.abort();
transaction_context.after_rollback();
return 0;
}
@ -315,75 +339,82 @@ private:
}
void wait_for_replayers(trrep::unique_lock<trrep::mutex>&) const override
{ }
void override_error(const trrep::client_error&) override { }
bool killed() const override { return false; }
void abort() const override { ::abort(); }
void store_globals() override { }
void debug_sync(const std::string&) override { }
void debug_suicide(const std::string&) override { }
void on_error(enum trrep::client_error) override { }
void run_one_transaction()
template <class Func>
int client_command(Func f)
{
int err(before_command());
dbms_storage_engine::transaction se_trx(server_.storage_engine());
if (err == 0)
{
err = before_statement();
if (err == 0)
{
err = start_transaction(server_.next_transaction_id());
se_trx.start(&transaction());
err = f();
}
after_statement();
}
after_command();
std::this_thread::sleep_for(std::chrono::milliseconds(10));
for (int i(0); i < 1 && err == 0; ++i)
{
std::ostringstream os;
err = before_command();
if (err == 0)
{
err = before_statement();
if (err == 0)
{
int data(std::rand() % 10000000);
os << data;
trrep::key key;
key.append_key_part("dbms", 4);
wsrep_conn_id_t client_id(id().get());
key.append_key_part(&client_id, sizeof(client_id));
key.append_key_part(&data, sizeof(data));
err = append_key(key);
err = append_data(trrep::data(os.str().c_str(), os.str().size()));
}
after_statement();
}
after_command();
}
std::this_thread::sleep_for(std::chrono::milliseconds(10));
if (err == 0)
{
err = before_command();
if (err == 0)
{
err = before_statement();
return err;
}
if (err == 0 && do_2pc())
void run_one_transaction()
{
reset_error();
int err = client_command(
[&]()
{
err = start_transaction(server_.next_transaction_id());
assert(err == 0);
se_trx_.start(&transaction());
return err;
});
err = err || current_error();
err = err || client_command(
[&]()
{
assert(transaction().active());
assert(err == 0);
int data(std::rand() % 10000000);
std::ostringstream os;
os << data;
trrep::key key;
key.append_key_part("dbms", 4);
wsrep_conn_id_t client_id(id().get());
key.append_key_part(&client_id, sizeof(client_id));
key.append_key_part(&data, sizeof(data));
err = append_key(key);
err = err || append_data(trrep::data(os.str().c_str(),
os.str().size()));
return err;
});
err = err || current_error();
err = err || client_command(
[&]()
{
assert(err == 0);
if (do_2pc())
{
err = err || before_prepare();
err = err || after_prepare();
}
err = err || before_commit();
se_trx_.commit();
err = err || ordered_commit();
err = err || after_commit();
after_statement();
}
after_command();
}
assert(transaction().state() == trrep::transaction_context::s_committed
||
transaction().state() == trrep::transaction_context::s_aborted);
return err;
});
assert((current_error() &&
transaction().state() == trrep::transaction_context::s_aborted) ||
transaction().state() == trrep::transaction_context::s_committed);
assert(se_trx_.active() == false);
assert(transaction().active() == false);
}
void report_progress(size_t i) const
@ -397,6 +428,7 @@ private:
}
trrep::default_mutex mutex_;
dbms_server& server_;
dbms_storage_engine::transaction se_trx_;
const size_t n_transactions_;
};
@ -482,6 +514,7 @@ void dbms_simulator::start()
boost::filesystem::create_directory(dir);
dbms_server& server(*it.first->second);
server.debug_log_level(params_.debug_log_level);
std::string server_options(params_.wsrep_provider_options);
if (server.load_provider(params_.wsrep_provider, server_options))
@ -622,7 +655,9 @@ int main(int argc, char** argv)
("clients", po::value<size_t>(&params.n_clients)->required(),
"number of clients to start per server")
("transactions", po::value<size_t>(&params.n_transactions),
"number of transactions run by a client");
"number of transactions run by a client")
("debug-log-level", po::value<int>(&params.debug_log_level),
"debug logging level: 0 - none, 1 - verbose");
po::variables_map vm;
po::store(po::parse_command_line(argc, argv, desc), vm);
po::notify(vm);

View File

@ -6,6 +6,7 @@
#define TRREP_EXCEPTION_HPP
#include <stdexcept>
#include <cstdlib>
namespace trrep
{
@ -14,7 +15,9 @@ namespace trrep
public:
runtime_error(const std::string& msg)
: std::runtime_error(msg)
{ }
{
::abort();
}
};
class not_implemented_error : public std::exception
@ -22,7 +25,9 @@ namespace trrep
public:
not_implemented_error()
: std::exception()
{ }
{
::abort();
}
};
}

View File

@ -16,13 +16,14 @@ namespace trrep
class log
{
public:
log()
: oss_()
log(const std::string& prefix = "INFO")
: prefix_(prefix)
, oss_()
{ }
~log()
{
trrep::unique_lock<trrep::mutex> lock(mutex_);
os_ << oss_.str() << "\n";
os_ << prefix_ << ": " << oss_.str() << "\n";
}
template <typename T>
std::ostream& operator<<(const T& val)
@ -30,10 +31,19 @@ namespace trrep
return (oss_ << val);
}
private:
const std::string prefix_;
std::ostringstream oss_;
static trrep::mutex& mutex_;
static std::ostream& os_;
};
class log_debug : public log
{
public:
log_debug()
: log("DEBUG") { }
};
}
#endif // TRREP_LOGGER_HPP

View File

@ -47,7 +47,6 @@ namespace trrep
}
void wait_for_replayers(trrep::unique_lock<trrep::mutex>&) const
TRREP_OVERRIDE { }
void override_error(const trrep::client_error&) TRREP_OVERRIDE { }
bool killed() const TRREP_OVERRIDE { return false; }
void abort() const TRREP_OVERRIDE { }
void store_globals() TRREP_OVERRIDE { }
@ -56,7 +55,7 @@ namespace trrep
{
::abort();
}
void on_error(enum trrep::client_error) { }
// Mock state modifiers
void fail_next_applying(bool fail_next_applying)
{ fail_next_applying_ = fail_next_applying; }

View File

@ -335,6 +335,9 @@ namespace trrep
const trrep::client_context& client_context,
const trrep::transaction_context& transaction_context) const;
void debug_log_level(int level) { debug_log_level_ = level; }
int debug_log_level() const { return debug_log_level_; }
protected:
/*! Server Context constructor
*
@ -365,6 +368,7 @@ namespace trrep
, address_(address)
, working_dir_(working_dir)
, rollback_mode_(rollback_mode)
, debug_log_level_(0)
{ }
private:
@ -384,6 +388,7 @@ namespace trrep
std::string address_;
std::string working_dir_;
enum rollback_mode rollback_mode_;
int debug_log_level_;
};
static inline std::string to_string(enum trrep::server_context::state state)

View File

@ -22,6 +22,8 @@ trrep::transaction_context::transaction_context(
, id_(transaction_id::invalid())
, state_(s_executing)
, state_hist_()
, bf_abort_state_(s_executing)
, bf_abort_client_state_()
, ws_handle_()
, trx_meta_()
, flags_()
@ -42,6 +44,7 @@ int trrep::transaction_context::start_transaction(
assert(active() == false);
id_ = id;
state_ = s_executing;
state_hist_.clear();
ws_handle_.trx_id = id_.get();
flags_ |= WSREP_FLAG_TRX_START;
switch (client_context_.mode())
@ -96,6 +99,7 @@ int trrep::transaction_context::before_prepare()
if (state() == s_must_abort)
{
assert(client_context_.mode() == trrep::client_context::m_replicating);
client_context_.override_error(trrep::e_deadlock_error);
return 1;
}
@ -143,6 +147,7 @@ int trrep::transaction_context::after_prepare()
if (state() == s_must_abort)
{
assert(client_context_.mode() == trrep::client_context::m_replicating);
client_context_.override_error(trrep::e_deadlock_error);
return 1;
}
@ -218,12 +223,17 @@ int trrep::transaction_context::before_commit()
if (ret == 0)
{
lock.unlock();
switch(provider_.commit_order_enter(&ws_handle_, &trx_meta_))
wsrep_status_t status(provider_.commit_order_enter(&ws_handle_, &trx_meta_));
lock.lock();
switch (status)
{
case WSREP_OK:
break;
case WSREP_BF_ABORT:
state(lock, s_must_abort);
if (state() != s_must_abort)
{
state(lock, s_must_abort);
}
ret = 1;
break;
default:
@ -231,7 +241,7 @@ int trrep::transaction_context::before_commit()
assert(0);
break;
}
lock.lock();
}
break;
case trrep::client_context::m_applier:
@ -405,6 +415,7 @@ int trrep::transaction_context::after_statement()
break;
case s_must_abort:
case s_cert_failed:
client_context_.override_error(trrep::e_deadlock_error);
lock.unlock();
ret = client_context_.rollback(*this);
lock.lock();
@ -453,44 +464,64 @@ bool trrep::transaction_context::bf_abort(
{
bool ret(false);
assert(lock.owns_lock());
switch (state())
assert(&lock.mutex() == &mutex());
if (active() == false)
{
case s_executing:
case s_preparing:
case s_certifying:
case s_committing:
trrep::log() << "Transaction not active, skipping bf abort";
}
else if (ordered() && seqno() < txc.seqno())
{
wsrep_seqno_t victim_seqno(WSREP_SEQNO_UNDEFINED);
wsrep_status_t status(client_context_.provider().bf_abort(
txc.seqno(), id_.get(), &victim_seqno));
switch (status)
trrep::log() << "Not allowed to BF abort transaction ordered before "
<< "aborter: " << seqno() << " < " << txc.seqno();
}
else
{
switch (state())
{
case WSREP_OK:
trrep::log() << "Seqno " << txc.seqno()
<< " succesfully BF aborted " << id_.get()
<< " victim_seqno " << victim_seqno;
state(lock, s_must_abort);
ret = true;
break;
default:
trrep::log() << "Seqno " << txc.seqno()
<< " failed to BF abort " << id_.get()
<< " with status " << status
<< " victim_seqno " << victim_seqno;
case s_executing:
case s_preparing:
case s_certifying:
case s_committing:
{
wsrep_seqno_t victim_seqno(WSREP_SEQNO_UNDEFINED);
wsrep_status_t status(client_context_.provider().bf_abort(
txc.seqno(), id_.get(), &victim_seqno));
switch (status)
{
case WSREP_OK:
trrep::log() << "Seqno " << txc.seqno()
<< " succesfully BF aborted " << id_.get()
<< " victim_seqno " << victim_seqno;
bf_abort_state_ = state();
state(lock, s_must_abort);
ret = true;
break;
default:
trrep::log() << "Seqno " << txc.seqno()
<< " failed to BF abort " << id_.get()
<< " with status " << status
<< " victim_seqno " << victim_seqno;
break;
}
break;
}
default:
trrep::log() << "BF abort not allowed in state "
<< trrep::to_string(state());
break;
}
break;
}
default:
trrep::log() << "BF abort not allowed in state "
<< trrep::to_string(state()) << "\n";
break;
}
if (client_context_.server_context().rollback_mode() == trrep::server_context::rm_sync)
if (ret)
{
//! \todo Launch background rollbacker.
assert(0);
bf_abort_client_state_ = client_context_.state();
if (client_context_.server_context().rollback_mode() ==
trrep::server_context::rm_sync)
{
//! \todo Launch background rollbacker.
assert(0);
}
}
return ret;
}
@ -761,7 +792,8 @@ void trrep::transaction_context::cleanup()
{
state_ = s_executing;
}
state_hist_.clear();
// Keep the state history for troubleshooting. Reset at start_transaction().
// state_hist_.clear();
trx_meta_.gtid = WSREP_GTID_UNDEFINED;
trx_meta_.stid.node = WSREP_UUID_UNDEFINED;
trx_meta_.stid.trx = trrep::transaction_id::invalid();
@ -774,11 +806,14 @@ void trrep::transaction_context::cleanup()
void trrep::transaction_context::debug_log_state(
const std::string& context TRREP_UNUSED) const
{
#if 0
trrep::log() << context
<< ": server: " << client_context_.server_context().name()
<< ": client: " << client_context_.id().get()
<< " trx: " << int64_t(id_.get())
<< " state: " << trrep::to_string(state_);
#endif /* 0 */
if (client_context_.debug_log_level() >= 1)
{
trrep::log_debug() << context
<< ": server: " << client_context_.server_context().name()
<< ": client: " << client_context_.id().get()
<< " trx: " << int64_t(id_.get())
<< " state: " << trrep::to_string(state_)
<< " error: "
<< trrep::to_string(client_context_.current_error());
}
}

View File

@ -157,6 +157,8 @@ namespace trrep
trrep::transaction_id id_;
enum state state_;
std::vector<enum state> state_hist_;
enum state bf_abort_state_;
int bf_abort_client_state_;
wsrep_ws_handle_t ws_handle_;
wsrep_trx_meta_t trx_meta_;
uint32_t flags_;

View File

@ -49,7 +49,7 @@ BOOST_AUTO_TEST_CASE(transaction_context_1pc)
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(tc.ordered() == false);
BOOST_REQUIRE(tc.certified() == false);
BOOST_REQUIRE(cc.current_error() == trrep::e_success);
}
//
@ -97,6 +97,7 @@ BOOST_AUTO_TEST_CASE(transaction_context_2pc)
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(tc.ordered() == false);
BOOST_REQUIRE(tc.certified() == false);
BOOST_REQUIRE(cc.current_error() == trrep::e_success);
}
//
@ -133,7 +134,7 @@ BOOST_AUTO_TEST_CASE(transaction_context_rollback)
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(tc.ordered() == false);
BOOST_REQUIRE(tc.certified() == false);
BOOST_REQUIRE(cc.current_error() == trrep::e_success);
}
//
@ -175,6 +176,7 @@ BOOST_AUTO_TEST_CASE(transaction_context_1pc_bf_before_before_commit)
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(tc.ordered() == false);
BOOST_REQUIRE(tc.certified() == false);
BOOST_REQUIRE(cc.current_error());
}
//
@ -216,6 +218,7 @@ BOOST_AUTO_TEST_CASE(transaction_context_2pc_bf_before_before_prepare)
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(tc.ordered() == false);
BOOST_REQUIRE(tc.certified() == false);
BOOST_REQUIRE(cc.current_error());
}
//
@ -261,6 +264,7 @@ BOOST_AUTO_TEST_CASE(transaction_context_2pc_bf_before_after_prepare)
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(tc.ordered() == false);
BOOST_REQUIRE(tc.certified() == false);
BOOST_REQUIRE(cc.current_error());
}
//
@ -303,8 +307,37 @@ BOOST_AUTO_TEST_CASE(transaction_context_1pc_bf_during_before_commit_uncertified
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(tc.ordered() == false);
BOOST_REQUIRE(tc.certified() == false);
BOOST_REQUIRE(cc.current_error());
}
//
// Test a transaction which gets BF aborted before after_statement.
//
BOOST_AUTO_TEST_CASE(transaction_context_1pc_bf_during_before_after_statement)
{
trrep::mock_server_context sc("s1", "s1",
trrep::server_context::rm_sync);
trrep::mock_client_context cc(sc, trrep::client_id(1), trrep::client_context::m_replicating);
trrep::transaction_context& tc(cc.transaction());
// Verify initial state
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(tc.state() == trrep::transaction_context::s_executing);
// Start a new transaction with ID 1
tc.start_transaction(1);
BOOST_REQUIRE(tc.active());
BOOST_REQUIRE(tc.id() == trrep::transaction_id(1));
BOOST_REQUIRE(tc.state() == trrep::transaction_context::s_executing);
trrep_mock::bf_abort_unordered(cc, tc);
BOOST_REQUIRE(tc.after_statement() == 0);
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(tc.ordered() == false);
BOOST_REQUIRE(tc.certified() == false);
BOOST_REQUIRE(cc.current_error());
}
//
// Test a 1PC transaction which gets BF aborted during before_commit via
@ -346,6 +379,7 @@ BOOST_AUTO_TEST_CASE(transaction_context_1pc_bf_during_before_commit_certified)
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(tc.ordered() == false);
BOOST_REQUIRE(tc.certified() == false);
BOOST_REQUIRE(cc.current_error() == trrep::e_success);
}
BOOST_AUTO_TEST_CASE(transaction_context_1pc_applying)
@ -374,6 +408,7 @@ BOOST_AUTO_TEST_CASE(transaction_context_1pc_applying)
BOOST_REQUIRE(tc.after_statement() == 0);
BOOST_REQUIRE(tc.state() == trrep::transaction_context::s_committed);
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(cc.current_error() == trrep::e_success);
}
BOOST_AUTO_TEST_CASE(transaction_context_2pc_applying)
@ -406,6 +441,7 @@ BOOST_AUTO_TEST_CASE(transaction_context_2pc_applying)
BOOST_REQUIRE(tc.after_statement() == 0);
BOOST_REQUIRE(tc.state() == trrep::transaction_context::s_committed);
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(cc.current_error() == trrep::e_success);
}
BOOST_AUTO_TEST_CASE(transaction_context_applying_rollback)
@ -432,5 +468,5 @@ BOOST_AUTO_TEST_CASE(transaction_context_applying_rollback)
BOOST_REQUIRE(tc.after_statement() == 0);
BOOST_REQUIRE(tc.state() == trrep::transaction_context::s_aborted);
BOOST_REQUIRE(tc.active() == false);
BOOST_REQUIRE(cc.current_error() == trrep::e_success);
}