diff --git a/dbsim/CMakeLists.txt b/dbsim/CMakeLists.txt index 012200b..aee964c 100644 --- a/dbsim/CMakeLists.txt +++ b/dbsim/CMakeLists.txt @@ -13,6 +13,7 @@ add_executable(dbsim db_simulator.cpp db_storage_engine.cpp db_threads.cpp + db_tls.cpp dbsim.cpp ) diff --git a/dbsim/db_params.cpp b/dbsim/db_params.cpp index 3524eeb..4fbefd0 100644 --- a/dbsim/db_params.cpp +++ b/dbsim/db_params.cpp @@ -87,6 +87,11 @@ db::params db::parse_args(int argc, char** argv) po::value(¶ms.cond_checks), "Enable checks for correct condition variable use. " " Effective only if thread-instrumentation is enabled") + ("tls-service", + po::value(¶ms.tls_service), + "Configure TLS service stubs.\n0 default disabled\n1 enabled\n" + "2 enabled with short read/write and renegotiation simulation\n" + "3 enabled with error simulation.") ; try { diff --git a/dbsim/db_params.hpp b/dbsim/db_params.hpp index c97f4d3..6443e13 100644 --- a/dbsim/db_params.hpp +++ b/dbsim/db_params.hpp @@ -42,6 +42,7 @@ namespace db int fast_exit; int thread_instrumentation; bool cond_checks; + int tls_service; params() : n_servers(0) , n_clients(0) @@ -58,6 +59,7 @@ namespace db , fast_exit(0) , thread_instrumentation() , cond_checks() + , tls_service() { } }; diff --git a/dbsim/db_simulator.cpp b/dbsim/db_simulator.cpp index 9b639b7..0971b11 100644 --- a/dbsim/db_simulator.cpp +++ b/dbsim/db_simulator.cpp @@ -20,6 +20,7 @@ #include "db_simulator.hpp" #include "db_client.hpp" #include "db_threads.hpp" +#include "db_tls.hpp" #include "wsrep/logger.hpp" @@ -27,6 +28,7 @@ #include static db::ti thread_instrumentation; +static db::tls tls_service; void db::simulator::run() { @@ -36,6 +38,7 @@ void db::simulator::run() std::cout << "Results:\n"; std::cout << stats() << std::endl; std::cout << db::ti::stats() << std::endl; + std::cout << db::tls::stats() << std::endl; } void db::simulator::sst(db::server& server, @@ -114,6 +117,7 @@ void db::simulator::start() { thread_instrumentation.level(params_.thread_instrumentation); thread_instrumentation.cond_checks(params_.cond_checks); + tls_service.init(params_.tls_service); wsrep::log_info() << "Provider: " << params_.wsrep_provider; std::string cluster_address(build_cluster_address()); @@ -149,6 +153,9 @@ void db::simulator::start() services.thread_service = params_.thread_instrumentation ? &thread_instrumentation : nullptr; + services.tls_service = params_.tls_service + ? &tls_service + : nullptr; if (server.server_state().load_provider(params_.wsrep_provider, server_options, services)) { diff --git a/dbsim/db_threads.cpp b/dbsim/db_threads.cpp index a2166a2..d066c65 100644 --- a/dbsim/db_threads.cpp +++ b/dbsim/db_threads.cpp @@ -35,6 +35,7 @@ #include #include +extern "C" { static void* start_thread(void* args_ptr); } namespace { struct ti_obj @@ -186,7 +187,6 @@ namespace } }; - void* thread_start_fn(void* args_ptr); class ti_thread : public ti_obj { @@ -209,7 +209,7 @@ namespace int run(void* (*fn)(void *), void* args) { auto ta(new thread_args{this, fn, args}); - return pthread_create(&th_, nullptr, thread_start_fn, ta); + return pthread_create(&th_, nullptr, start_thread, ta); } int detach() @@ -254,24 +254,6 @@ namespace bool detached_; }; - void* thread_start_fn(void* args_ptr) - { - thread_args* ta(reinterpret_cast(args_ptr)); - ti_thread* thread = reinterpret_cast(ta->this_thread); - pthread_setspecific(this_thread_key, thread); - void* (*fn)(void*) = ta->fn; - void* args = ta->args; - delete ta; - void* ret((*fn)(args)); - pthread_setspecific(this_thread_key, nullptr); - - // If we end here the thread returned instead of calling - // pthread_exit() - if (thread->detached()) - delete thread; - return ret; - } - class ti_mutex : public ti_obj { public: @@ -482,6 +464,42 @@ int db::ti::after_init() // Thread // ////////////////////////////////////////////////////////////////////////////// +extern "C" +{ +static void* start_thread(void* args_ptr) +{ + thread_args* ta(reinterpret_cast(args_ptr)); + ti_thread* thread = reinterpret_cast(ta->this_thread); + pthread_setspecific(this_thread_key, thread); + void* (*fn)(void*) = ta->fn; + void* args = ta->args; + delete ta; + void* ret = (*fn)(args); + pthread_setspecific(this_thread_key, nullptr); + // If we end here the thread returned instead of calling + // pthread_exit() + if (thread->detached()) + delete thread; + return ret; +} + +WSREP_NORETURN +static void exit_thread(wsrep::thread_service::thread* thread, void* retval) +{ + pthread_setspecific(this_thread_key, nullptr); + ti_thread* th(reinterpret_cast(thread)); + th->retval(retval); + if (th->detached()) + delete th; + pthread_exit(retval); +} +} // extern "C" + +db::ti::ti() +{ + thread_service::exit = exit_thread; +} + const wsrep::thread_service::thread_key* db::ti::create_thread_key(const char* name) WSREP_NOEXCEPT { @@ -513,15 +531,6 @@ int db::ti::detach(wsrep::thread_service::thread* thread) WSREP_NOEXCEPT return reinterpret_cast(thread)->detach(); } -void db::ti::exit(wsrep::thread_service::thread* thread, void* retval) WSREP_NOEXCEPT -{ - ti_thread* th(reinterpret_cast(thread)); - th->retval(retval); - if (th->detached()) - delete th; - pthread_exit(retval); -} - int db::ti::equal(wsrep::thread_service::thread* thread_1, wsrep::thread_service::thread* thread_2) WSREP_NOEXCEPT { diff --git a/dbsim/db_threads.hpp b/dbsim/db_threads.hpp index 01ad956..32d44c1 100644 --- a/dbsim/db_threads.hpp +++ b/dbsim/db_threads.hpp @@ -28,7 +28,7 @@ namespace db class ti : public wsrep::thread_service { public: - ti() { } + ti(); int before_init() override; int after_init() override; @@ -41,7 +41,6 @@ namespace db int detach(wsrep::thread_service::thread*) WSREP_NOEXCEPT override; int equal(wsrep::thread_service::thread*, wsrep::thread_service::thread*) WSREP_NOEXCEPT override; - void exit(wsrep::thread_service::thread*, void*) WSREP_NOEXCEPT override; int join(wsrep::thread_service::thread*, void**) WSREP_NOEXCEPT override; wsrep::thread_service::thread* self() WSREP_NOEXCEPT override; int setschedparam(wsrep::thread_service::thread*, int, diff --git a/dbsim/db_tls.cpp b/dbsim/db_tls.cpp new file mode 100644 index 0000000..c242668 --- /dev/null +++ b/dbsim/db_tls.cpp @@ -0,0 +1,451 @@ +/* + * Copyright (C) 2020 Codership Oy + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see . + */ + +/** @file db_tls.cpp + * + * This file demonstrates the use of TLS service. It does not implement + * real encryption, but may manipulate stream bytes for testing purposes. + */ + +#include "db_tls.hpp" + +#include "wsrep/logger.hpp" + +#include // read() +#include +#include +#include // send() +#include +#include +#include + +#include +#include + +namespace +{ + class db_stream : public wsrep::tls_stream + { + public: + db_stream(int fd, int mode) + : fd_(fd) + , state_(s_initialized) + , last_error_() + , mode_(mode) + , stats_() + , is_blocking_() + { + int val(fcntl(fd_, F_GETFL, 0)); + is_blocking_ = not (val & O_NONBLOCK); + } + struct stats + { + size_t bytes_read{0}; + size_t bytes_written{0}; + }; + + /* + * in idle --| + * |-> ch -| ^ | -> want_read --| + * |-> sh -| ---- |--> | -> want_write --| + * |----------------------| + */ + enum state + { + s_initialized, + s_client_handshake, + s_server_handshake, + s_idle, + s_want_read, + s_want_write + }; + + int get_error_number() const { return last_error_; } + const void* get_error_category() const + { + return reinterpret_cast(1); + } + + static char* get_error_message(int value, const void*) + { + return ::strerror(value); + } + + enum wsrep::tls_service::status client_handshake(); + + enum wsrep::tls_service::status server_handshake(); + + wsrep::tls_service::op_result read(void*, size_t); + + wsrep::tls_service::op_result write(const void*, size_t); + + enum state state() const { return state_; } + + int fd() const { return fd_; } + void inc_reads(size_t val) { stats_.bytes_read += val; } + void inc_writes(size_t val) { stats_.bytes_written += val; } + const stats& get_stats() const { return stats_; } + private: + enum wsrep::tls_service::status handle_handshake_read(const char* expect); + size_t determine_read_count(size_t max_count) + { + if (is_blocking_ || mode_ < 2) return max_count; + else if (::rand() % 100 == 0) return std::min(size_t(42), max_count); + else return max_count; + } + size_t determine_write_count(size_t count) + { + if (is_blocking_ || mode_ < 2) return count; + else if (::rand() % 100 == 0) return std::min(size_t(43), count); + else return count; + } + + ssize_t do_read(void* buf, size_t max_count) + { + if (is_blocking_ || mode_ < 3 ) + return ::read(fd_, buf, max_count); + else if (::rand() % 1000 == 0) return EINTR; + else return ::read(fd_, buf, max_count); + } + + ssize_t do_write(const void* buf, size_t count) + { + if (is_blocking_ || mode_ < 3) + return ::send(fd_, buf, count, MSG_NOSIGNAL); + else if (::rand() % 1000 == 0) return EINTR; + else return ::send(fd_, buf, count, MSG_NOSIGNAL); + } + + wsrep::tls_service::op_result map_success(ssize_t result) + { + if (is_blocking_ || mode_ < 2) + { + return wsrep::tls_service::op_result{ + wsrep::tls_service::success, size_t(result)}; + } + else if (::rand() % 1000 == 0) + { + wsrep::log_info() << "Success want extra read"; + state_ = s_want_read; + return wsrep::tls_service::op_result{ + wsrep::tls_service::want_read, size_t(result)}; + } + else if (::rand() % 1000 == 0) + { + wsrep::log_info() << "Success want extra write"; + state_ = s_want_write; + return wsrep::tls_service::op_result{ + wsrep::tls_service::want_write, size_t(result)}; + } + else + { + return wsrep::tls_service::op_result{ + wsrep::tls_service::success, size_t(result)}; + } + } + + wsrep::tls_service::op_result map_result(ssize_t result) + { + if (result > 0) + { + return map_success(result); + } + else if (result == 0) + { + return wsrep::tls_service::op_result{ + wsrep::tls_service::eof, 0}; + } + else if (errno == EAGAIN || errno == EWOULDBLOCK) + { + return wsrep::tls_service::op_result{ + wsrep::tls_service::want_read, 0}; + } + else + { + last_error_ = errno; + return wsrep::tls_service::op_result{ + wsrep::tls_service::error, 0}; + } + } + + void clear_error() { last_error_ = 0; } + + int fd_; + enum state state_; + int last_error_; + // Operation mode: + // 1 - simulate handshake exchange + // 2 - simulate errors and short reads + int mode_; + stats stats_; + bool is_blocking_; + }; + + enum wsrep::tls_service::status db_stream::client_handshake() + { + clear_error(); + enum wsrep::tls_service::status ret; + assert(state_ == s_initialized || + state_ == s_client_handshake || + state_ == s_want_write); + if (state_ == s_initialized) + { + (void)::send(fd_, "clie", 4, MSG_NOSIGNAL); + ret = wsrep::tls_service::want_read; + state_ = s_client_handshake; + wsrep::log_info() << this << " client handshake sent"; + stats_.bytes_written += 4; + if (not is_blocking_) return ret; + } + + if (state_ == s_client_handshake) + { + if ((ret = handle_handshake_read("serv")) == + wsrep::tls_service::success) + { + state_ = s_want_write; + ret = wsrep::tls_service::want_write; + } + if (not is_blocking_) return ret; + } + + if (state_ == s_want_write) + { + state_ = s_idle; + ret = wsrep::tls_service::success; + if (not is_blocking_) return ret; + } + + if (not is_blocking_) + { + last_error_ = EPROTO; + ret = wsrep::tls_service::error; + } + return ret; + } + + + + enum wsrep::tls_service::status db_stream::server_handshake() + { + enum wsrep::tls_service::status ret; + assert(state_ == s_initialized || + state_ == s_server_handshake || + state_ == s_want_write); + + if (state_ == s_initialized) + { + ::send(fd_, "serv", 4, MSG_NOSIGNAL); + ret = wsrep::tls_service::want_read; + state_ = s_server_handshake; + stats_.bytes_written += 4; + if (not is_blocking_) return ret; + } + + if (state_ == s_server_handshake) + { + if ((ret = handle_handshake_read("clie")) == + wsrep::tls_service::success) + { + state_ = s_want_write; + ret = wsrep::tls_service::want_write; + } + if (not is_blocking_) return ret; + } + + if (state_ == s_want_write) + { + state_ = s_idle; + ret = wsrep::tls_service::success; + if (not is_blocking_) return ret; + } + + if (not is_blocking_) + { + last_error_ = EPROTO; + ret = wsrep::tls_service::error; + } + return ret; + } + + enum wsrep::tls_service::status db_stream::handle_handshake_read( + const char* expect) + { + assert(::strlen(expect) >= 4); + char buf[4] = { }; + ssize_t read_result(::read(fd_, buf, sizeof(buf))); + if (read_result > 0) stats_.bytes_read += size_t(read_result); + enum wsrep::tls_service::status ret; + if (read_result == -1 && + (errno == EWOULDBLOCK || errno == EAGAIN)) + { + ret = wsrep::tls_service::want_read; + } + else if (read_result == 0) + { + ret = wsrep::tls_service::eof; + } + else if (read_result != 4 || ::memcmp(buf, expect, 4)) + { + last_error_ = EPROTO; + ret = wsrep::tls_service::error; + } + else + { + wsrep::log_info() << "Handshake success: " << std::string(buf, 4); + ret = wsrep::tls_service::success; + } + return ret; + } + + wsrep::tls_service::op_result db_stream::read(void* buf, size_t max_count) + { + clear_error(); + if (state_ == s_want_read) + { + state_ = s_idle; + if (max_count == 0) + return wsrep::tls_service::op_result{ + wsrep::tls_service::success, 0}; + } + max_count = determine_read_count(max_count); + ssize_t read_result(do_read(buf, max_count)); + if (read_result > 0) + { + inc_reads(size_t(read_result)); + } + return map_result(read_result); + } + + wsrep::tls_service::op_result db_stream::write( + const void* buf, size_t count) + { + clear_error(); + if (state_ == s_want_write) + { + state_ = s_idle; + if (count == 0) + return wsrep::tls_service::op_result{ + wsrep::tls_service::success, 0}; + } + count = determine_write_count(count); + ssize_t write_result(do_write(buf, count)); + if (write_result > 0) + { + inc_writes(size_t(write_result)); + } + return map_result(write_result); + } +} + + +static db_stream::stats global_stats; +std::mutex global_stats_lock; +static int global_mode; + +static void merge_to_global_stats(const db_stream::stats& stats) +{ + std::lock_guard lock(global_stats_lock); + global_stats.bytes_read += stats.bytes_read; + global_stats.bytes_written += stats.bytes_written; +} + +wsrep::tls_stream* db::tls::create_tls_stream(int fd) WSREP_NOEXCEPT +{ + auto ret(new db_stream(fd, global_mode)); + wsrep::log_debug() << "New DB stream: " << ret; + return ret; +} + +void db::tls::destroy(wsrep::tls_stream* stream) WSREP_NOEXCEPT +{ + auto dbs(static_cast(stream)); + merge_to_global_stats(dbs->get_stats()); + wsrep::log_debug() << "Stream destroy: " << dbs->get_stats().bytes_read + << " " << dbs->get_stats().bytes_written; + wsrep::log_debug() << "Stream destroy" << dbs; + delete dbs; +} + +int db::tls::get_error_number(const wsrep::tls_stream* stream) + const WSREP_NOEXCEPT +{ + return static_cast(stream)->get_error_number(); +} + +const void* db::tls::get_error_category(const wsrep::tls_stream* stream) + const WSREP_NOEXCEPT +{ + return static_cast(stream)->get_error_category(); +} + +const char* db::tls::get_error_message(const wsrep::tls_stream*, + int value, const void* category) + const WSREP_NOEXCEPT +{ + return db_stream::get_error_message(value, category); +} + +enum wsrep::tls_service::status +db::tls::client_handshake(wsrep::tls_stream* stream) WSREP_NOEXCEPT +{ + return static_cast(stream)->client_handshake(); +} + +enum wsrep::tls_service::status +db::tls::server_handshake(wsrep::tls_stream* stream) WSREP_NOEXCEPT +{ + return static_cast(stream)->server_handshake(); +} + +wsrep::tls_service::op_result db::tls::read( + wsrep::tls_stream* stream, + void* buf, size_t max_count) WSREP_NOEXCEPT +{ + return static_cast(stream)->read(buf, max_count); +} + +wsrep::tls_service::op_result db::tls::write( + wsrep::tls_stream* stream, + const void* buf, size_t count) WSREP_NOEXCEPT +{ + return static_cast(stream)->write(buf, count); +} + +wsrep::tls_service::status +db::tls::shutdown(wsrep::tls_stream*) WSREP_NOEXCEPT +{ + // @todo error simulation + return wsrep::tls_service::success; +} + + +void db::tls::init(int mode) +{ + global_mode = mode; +} + +std::string db::tls::stats() +{ + std::ostringstream oss; + oss << "Transport stats:\n" + << " bytes_read: " << global_stats.bytes_read << "\n" + << " bytes_written: " << global_stats.bytes_written << "\n"; + return oss.str(); +} diff --git a/dbsim/db_tls.hpp b/dbsim/db_tls.hpp new file mode 100644 index 0000000..97c4387 --- /dev/null +++ b/dbsim/db_tls.hpp @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2020 Codership Oy + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see . + */ + +#ifndef WSREP_DB_TLS_HPP +#define WSREP_DB_TLS_HPP + +#include "wsrep/tls_service.hpp" + +#include + +namespace db +{ + class tls : public wsrep::tls_service + { + public: + virtual wsrep::tls_stream* create_tls_stream(int) + WSREP_NOEXCEPT override; + virtual void destroy(wsrep::tls_stream*) WSREP_NOEXCEPT override; + virtual int get_error_number(const wsrep::tls_stream*) const + WSREP_NOEXCEPT override; + virtual const void* get_error_category(const wsrep::tls_stream*) const + WSREP_NOEXCEPT override; + virtual const char* get_error_message(const wsrep::tls_stream*, + int, const void*) const + WSREP_NOEXCEPT override; + virtual enum wsrep::tls_service::status + client_handshake(wsrep::tls_stream*) + WSREP_NOEXCEPT override; + virtual enum wsrep::tls_service::status + server_handshake(wsrep::tls_stream*) + WSREP_NOEXCEPT override; + virtual wsrep::tls_service::op_result + read(wsrep::tls_stream*, void* buf, size_t max_count) + WSREP_NOEXCEPT override; + virtual wsrep::tls_service::op_result + write(wsrep::tls_stream*, const void* buf, size_t count) + WSREP_NOEXCEPT override; + virtual wsrep::tls_service::status + shutdown(wsrep::tls_stream*) WSREP_NOEXCEPT override; + + static void init(int mode); + static std::string stats(); + }; +}; + +#endif // WSREP_DB_TLS_HPP diff --git a/include/wsrep/provider.hpp b/include/wsrep/provider.hpp index a828a6b..4c85880 100644 --- a/include/wsrep/provider.hpp +++ b/include/wsrep/provider.hpp @@ -45,6 +45,8 @@ namespace wsrep class server_state; class high_priority_service; class thread_service; + class tls_service; + class stid { public: @@ -419,8 +421,10 @@ namespace wsrep struct services { wsrep::thread_service* thread_service; + wsrep::tls_service* tls_service; services() : thread_service() + , tls_service() { } }; diff --git a/include/wsrep/thread_service.hpp b/include/wsrep/thread_service.hpp index 1f8d2f0..702d71c 100644 --- a/include/wsrep/thread_service.hpp +++ b/include/wsrep/thread_service.hpp @@ -28,6 +28,9 @@ * condition variables. */ +#ifndef WSREP_THREAD_SERVICE_HPP +#define WSREP_THREAD_SERVICE_HPP + #include // size_t #include "compiler.hpp" @@ -41,13 +44,14 @@ namespace wsrep { public: + thread_service() : exit() { } virtual ~thread_service() { } - class thread_key { }; - class thread { }; - class mutex_key { }; - class mutex { }; - class cond_key { }; - class cond { }; + struct thread_key { }; + struct thread { }; + struct mutex_key { }; + struct mutex { }; + struct cond_key { }; + struct cond { }; /** * Method will be called before library side thread @@ -69,7 +73,12 @@ namespace wsrep = 0; virtual int detach(thread*) WSREP_NOEXCEPT = 0; virtual int equal(thread*, thread*) WSREP_NOEXCEPT = 0; - WSREP_NORETURN virtual void exit(thread*, void* retval) WSREP_NOEXCEPT = 0; + + /* + * This unlike others is a function pointer to + * avoid having C++ methods on thread exit codepath. + */ + WSREP_NORETURN void (*exit)(thread*, void* retval); virtual int join(thread*, void** retval) WSREP_NOEXCEPT = 0; virtual thread* self() WSREP_NOEXCEPT = 0; virtual int setschedparam(thread*, int, @@ -98,3 +107,5 @@ namespace wsrep virtual int broadcast(cond*) WSREP_NOEXCEPT = 0; }; } // namespace wsrep + +#endif // WSREP_THREAD_SERVICE_HPP diff --git a/include/wsrep/tls_service.hpp b/include/wsrep/tls_service.hpp new file mode 100644 index 0000000..07d2064 --- /dev/null +++ b/include/wsrep/tls_service.hpp @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2020 Codership Oy + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see . + */ + + +/** @file tls_service.hpp + * + * Service interface for interacting with DBMS provided + * TLS and encryption facilities. + */ + +#ifndef WSREP_TLS_SERVICE_HPP +#define WSREP_TLS_SERVICE_HPP + +#include "compiler.hpp" + +#include // ssize_t + +namespace wsrep +{ + + /* Type tags for TLS context and TLS stream. */ + struct tls_context { }; + struct tls_stream { }; + + /** @class tls_service + * + * TLS service interface. This provides an interface corresponding + * to wsrep-API TLS service. For details see wsrep-API/wsrep_tls_service.h + */ + class tls_service + { + public: + enum status + { + success = 0, + want_read, + want_write, + eof, + error + }; + + struct op_result + { + /** Status code of the operation of negative system error number. */ + ssize_t status; + /** Bytes transferred from/to given buffer during the operation. */ + size_t bytes_transferred; + }; + + virtual ~tls_service() { } + /** + * @return Zero on success, system error code on failure. + */ + virtual tls_stream* create_tls_stream(int fd) WSREP_NOEXCEPT = 0; + virtual void destroy(tls_stream*) WSREP_NOEXCEPT = 0; + + virtual int get_error_number(const tls_stream*) const WSREP_NOEXCEPT = 0; + virtual const void* get_error_category(const tls_stream*) const WSREP_NOEXCEPT = 0; + virtual const char* get_error_message(const tls_stream*, + int value, const void* category) + const WSREP_NOEXCEPT = 0; + /** + * @return Status enum. + */ + virtual status client_handshake(tls_stream*) WSREP_NOEXCEPT = 0; + + /** + * @return Status enum or negative error code. + */ + virtual status server_handshake(tls_stream*) WSREP_NOEXCEPT = 0; + + /** + * Read at most max_count bytes into buf. + */ + virtual op_result read(tls_stream*, + void* buf, size_t max_count) WSREP_NOEXCEPT = 0; + + /** + * Write at most count bytes from buf. + */ + virtual op_result write(tls_stream*, + const void* buf, size_t count) WSREP_NOEXCEPT = 0; + + /** + * Shutdown TLS stream. + */ + virtual status shutdown(tls_stream*) WSREP_NOEXCEPT = 0; + }; +} + +#endif // WSREP_TLS_SERVICE_HPP diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a49f95..0401494 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,6 +16,7 @@ add_library(wsrep-lib server_state.cpp thread.cpp thread_service_v1.cpp + tls_service_v1.cpp transaction.cpp uuid.cpp wsrep_provider_v26.cpp) diff --git a/src/service_helpers.hpp b/src/service_helpers.hpp new file mode 100644 index 0000000..6e9f9ca --- /dev/null +++ b/src/service_helpers.hpp @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2020 Codership Oy + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see . + */ + +#ifndef WSREP_SERVICE_HELPERS_HPP +#define WSREP_SERVICE_HELPERS_HPP + +#include "wsrep/logger.hpp" + +#include +#include + +namespace wsrep_impl +{ + template + int service_probe(void* dlh, const char* symbol, const char* service_name) + { + union { + InitFn dlfun; + void* obj; + } alias; + // Clear previous errors + (void)dlerror(); + alias.obj = dlsym(dlh, symbol); + if (alias.obj) + { + return 0; + } + else + { + wsrep::log_warning() << "Support for " << service_name + << " " << symbol + << " not found from provider: " + << dlerror(); + return ENOTSUP; + } + } + + + template + int service_init(void* dlh, + const char* symbol, + ServiceCallbacks service_callbacks, + const char* service_name) + { + union { + InitFn dlfun; + void* obj; + } alias; + // Clear previous errors + (void)dlerror(); + alias.obj = dlsym(dlh, symbol); + if (alias.obj) + { + wsrep::log_info() << "Initializing " << service_name; + return (*alias.dlfun)(service_callbacks); + } + else + { + wsrep::log_info() + << "Provider does not support " << service_name; + return ENOTSUP; + } + } + + template + void service_deinit(void* dlh, const char* symbol, const char* service_name) + { + union { + DeinitFn dlfun; + void* obj; + } alias; + // Clear previous errors + (void)dlerror(); + alias.obj = dlsym(dlh, symbol); + if (alias.obj) + { + wsrep::log_info() << "Deinitializing " << service_name; + (*alias.dlfun)(); + } + else + { + wsrep::log_info() + << "Provider does not support deinitializing of " + << service_name; + } + } +} + +#endif // WSREP_SERVICE_HELPERS_HPP + diff --git a/src/thread_service_v1.cpp b/src/thread_service_v1.cpp index 1d07af2..ee81ba3 100644 --- a/src/thread_service_v1.cpp +++ b/src/thread_service_v1.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019 Codership Oy + * Copyright (C) 2019-2020 Codership Oy * * This file is part of wsrep-lib. * @@ -18,6 +18,7 @@ */ #include "thread_service_v1.hpp" +#include "service_helpers.hpp" #include "wsrep/thread_service.hpp" #include "wsrep/logger.hpp" @@ -36,6 +37,7 @@ namespace wsrep_thread_service_v1 // Pointer to thread service implementation provided by // the application. static wsrep::thread_service* thread_service_impl{ 0 }; + static std::atomic use_count; static const wsrep_thread_key_t* thread_key_create_cb(const char* name) { @@ -236,50 +238,48 @@ namespace wsrep_thread_service_v1 int wsrep::thread_service_v1_probe(void* dlh) { - typedef int (*init_fn)(wsrep_thread_service_v1_t*); - union { - init_fn dlfun; - void* obj; - } alias; - // Clear previous errors - (void)dlerror(); - alias.obj = dlsym(dlh, WSREP_THREAD_SERVICE_INIT_FUNC); - if (alias.obj) + typedef int (*init_fn)(wsrep_thread_service_v1_t*); + typedef void (*deinit_fn)(); + if (wsrep_impl::service_probe( + dlh, WSREP_THREAD_SERVICE_INIT_FUNC_V1, "thread service v1") || + wsrep_impl::service_probe( + dlh, WSREP_THREAD_SERVICE_DEINIT_FUNC_V1, "thread service v1")) { - wsrep::log_info() - << "Found support for thread service v1 from provider"; - return 0; + wsrep::log_warning() << "Provider does not support thread service v1"; + return 1; } - else - { - wsrep::log_info() << "Thread service v1 not found from provider: " - << dlerror(); - return ENOTSUP; - } - + return 0; } int wsrep::thread_service_v1_init(void* dlh, wsrep::thread_service* thread_service) { if (not (dlh && thread_service)) return EINVAL; - typedef int (*init_fn)(wsrep_thread_service_v1_t*); - union { - init_fn dlfun; - void* obj; - } alias; - alias.obj = dlsym(dlh, WSREP_THREAD_SERVICE_INIT_FUNC); - if (alias.obj) + wsrep_thread_service_v1::thread_service_impl = thread_service; + int ret(0); + if ((ret = wsrep_impl::service_init( + dlh, WSREP_THREAD_SERVICE_INIT_FUNC_V1, + &wsrep_thread_service_v1::thread_service_callbacks, + "thread service v1"))) { - wsrep::log_info() << "Initializing process instrumentation"; - wsrep_thread_service_v1::thread_service_impl = thread_service; - return (*alias.dlfun)(&wsrep_thread_service_v1::thread_service_callbacks); + wsrep_thread_service_v1::thread_service_impl = 0; } else { - wsrep::log_info() - << "Provider does not support process instrumentation"; - return ENOTSUP; + ++wsrep_thread_service_v1::use_count; + } + return ret; +} + +void wsrep::thread_service_v1_deinit(void* dlh) +{ + typedef int (*deinit_fn)(); + wsrep_impl::service_deinit( + dlh, WSREP_THREAD_SERVICE_DEINIT_FUNC_V1, "thread service v1"); + --wsrep_thread_service_v1::use_count; + if (wsrep_thread_service_v1::use_count == 0) + { + wsrep_thread_service_v1::thread_service_impl = 0; } } diff --git a/src/thread_service_v1.hpp b/src/thread_service_v1.hpp index 6fd8b25..b830004 100644 --- a/src/thread_service_v1.hpp +++ b/src/thread_service_v1.hpp @@ -42,6 +42,14 @@ namespace wsrep */ int thread_service_v1_init(void* dlh, wsrep::thread_service* thread_service); + + /** + * Deinitialize the thread service. + * + * @params dlh Handler returned by dlopen(). + */ + void thread_service_v1_deinit(void* dlh); + } #endif // WSREP_THREAD_SERVICE_V1_HPP diff --git a/src/tls_service_v1.cpp b/src/tls_service_v1.cpp new file mode 100644 index 0000000..8746a76 --- /dev/null +++ b/src/tls_service_v1.cpp @@ -0,0 +1,232 @@ +/* + * Copyright (C) 2020 Codership Oy + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see . + */ + +#include "tls_service_v1.hpp" + +#include "wsrep/tls_service.hpp" +#include "wsrep/logger.hpp" +#include "v26/wsrep_tls_service.h" +#include "service_helpers.hpp" + +#include + +namespace wsrep_tls_service_v1 +{ + static wsrep::tls_service* tls_service_impl{0}; + static std::atomic use_count; + + static int tls_stream_init_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + stream->opaque = + tls_service_impl->create_tls_stream(stream->fd); + if (not stream->opaque) + { + return ENOMEM; + } + return 0; + } + + static void tls_stream_deinit_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + tls_service_impl->destroy( + reinterpret_cast(stream->opaque)); + } + + static int tls_stream_get_error_number_cb( + wsrep_tls_context_t*, + const wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + return tls_service_impl->get_error_number( + reinterpret_cast(stream->opaque)); + } + + static const void* tls_stream_get_error_category_cb( + wsrep_tls_context_t*, + const wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + return tls_service_impl->get_error_category( + reinterpret_cast(stream->opaque)); + } + + static const char* tls_error_message_get_cb( + wsrep_tls_context_t*, + const wsrep_tls_stream_t* stream, + int value, const void* category) + { + assert(tls_service_impl); + return tls_service_impl->get_error_message( + reinterpret_cast(stream->opaque), value, category); + } + + static enum wsrep_tls_result map_return_value(ssize_t status) + { + switch (status) + { + case wsrep::tls_service::success: + return wsrep_tls_result_success; + case wsrep::tls_service::want_read: + return wsrep_tls_result_want_read; + case wsrep::tls_service::want_write: + return wsrep_tls_result_want_write; + case wsrep::tls_service::eof: + return wsrep_tls_result_eof; + case wsrep::tls_service::error: + return wsrep_tls_result_error; + default: + assert(status < 0); + return wsrep_tls_result_error; + } + } + + + static enum wsrep_tls_result + tls_stream_client_handshake_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + return map_return_value( + tls_service_impl->client_handshake( + reinterpret_cast(stream->opaque))); + } + + static enum wsrep_tls_result + tls_stream_server_handshake_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + return map_return_value( + tls_service_impl->server_handshake( + reinterpret_cast(stream->opaque))); + } + + static enum wsrep_tls_result tls_stream_read_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream, + void* buf, + size_t max_count, + size_t* bytes_transferred) + { + assert(tls_service_impl); + auto result(tls_service_impl->read( + reinterpret_cast(stream->opaque), + buf, max_count)); + *bytes_transferred = result.bytes_transferred; + return map_return_value(result.status); + } + + static enum wsrep_tls_result tls_stream_write_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream, + const void* buf, + size_t count, + size_t* bytes_transferred) + { + assert(tls_service_impl); + auto result(tls_service_impl->write( + reinterpret_cast(stream->opaque), + buf, count)); + *bytes_transferred = result.bytes_transferred; + return map_return_value(result.status); + } + + static enum wsrep_tls_result + tls_stream_shutdown_cb(wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + // @todo Handle other values than success. + return map_return_value( + tls_service_impl->shutdown( + reinterpret_cast(stream->opaque))); + } + + static wsrep_tls_service_v1_t tls_service_callbacks = + { + tls_stream_init_cb, + tls_stream_deinit_cb, + tls_stream_get_error_number_cb, + tls_stream_get_error_category_cb, + tls_stream_client_handshake_cb, + tls_stream_server_handshake_cb, + tls_stream_read_cb, + tls_stream_write_cb, + tls_stream_shutdown_cb, + tls_error_message_get_cb, + 0 // we pass NULL context for now. + }; +} + +int wsrep::tls_service_v1_probe(void* dlh) +{ + typedef int (*init_fn)(wsrep_tls_service_v1_t*); + typedef void (*deinit_fn)(); + if (wsrep_impl::service_probe( + dlh, WSREP_TLS_SERVICE_INIT_FUNC_V1, "tls service v1") || + wsrep_impl::service_probe( + dlh, WSREP_TLS_SERVICE_DEINIT_FUNC_V1, "tls service v1")) + { + wsrep::log_warning() << "Provider does not support tls service v1"; + return 1; + } + return 0; +} + +int wsrep::tls_service_v1_init(void* dlh, + wsrep::tls_service* tls_service) +{ + if (not (dlh && tls_service)) return EINVAL; + + typedef int (*init_fn)(wsrep_tls_service_v1_t*); + wsrep_tls_service_v1::tls_service_impl = tls_service; + int ret(0); + if ((ret = wsrep_impl::service_init( + dlh, WSREP_TLS_SERVICE_INIT_FUNC_V1, + &wsrep_tls_service_v1::tls_service_callbacks, + "tls service v1"))) + { + wsrep_tls_service_v1::tls_service_impl = 0; + } + else + { + ++wsrep_tls_service_v1::use_count; + } + return ret; +} + +void wsrep::tls_service_v1_deinit(void* dlh) +{ + typedef int (*deinit_fn)(); + wsrep_impl::service_deinit( + dlh, WSREP_TLS_SERVICE_DEINIT_FUNC_V1, "tls service v1"); + --wsrep_tls_service_v1::use_count; + if (wsrep_tls_service_v1::use_count == 0) + { + wsrep_tls_service_v1::tls_service_impl = 0; + } +} diff --git a/src/tls_service_v1.hpp b/src/tls_service_v1.hpp new file mode 100644 index 0000000..ee9a14b --- /dev/null +++ b/src/tls_service_v1.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2020 Codership Oy + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see . + */ + +#ifndef WSREP_TLS_SERVICE_V1_HPP +#define WSREP_TLS_SERVICE_V1_HPP + +namespace wsrep +{ + class tls_service; + /** + * Probe thread_service_v1 support in loaded library. + * + * @param dlh Handle returned by dlopen(). + * + * @return Zero on success, non-zero system error code on failure. + */ + int tls_service_v1_probe(void *dlh); + + /** + * Initialize TLS service. + * + * @param dlh Handle returned by dlopen(). + * @params thread_service Pointer to wsrep::thread_service implementation. + * + * @return Zero on success, non-zero system error code on failure. + */ + int tls_service_v1_init(void* dlh, + wsrep::tls_service* thread_service); + + /** + * Deinitialize TLS service. + * + * @param dlh Handler returned by dlopen(). + */ + void tls_service_v1_deinit(void* dlh); +} + +#endif // WSREP_TLS_SERVICE_V1_HPP diff --git a/src/wsrep_provider_v26.cpp b/src/wsrep_provider_v26.cpp index 769067b..a579dff 100644 --- a/src/wsrep_provider_v26.cpp +++ b/src/wsrep_provider_v26.cpp @@ -26,8 +26,10 @@ #include "wsrep/exception.hpp" #include "wsrep/logger.hpp" #include "wsrep/thread_service.hpp" +#include "wsrep/tls_service.hpp" #include "thread_service_v1.hpp" +#include "tls_service_v1.hpp" #include "v26/wsrep_api.h" @@ -592,7 +594,7 @@ namespace if (wsrep::thread_service_v1_probe(dlh)) { // No support in library. - return 0; + return 1; } else { @@ -610,6 +612,58 @@ namespace } return 0; } + + static void deinit_thread_service(void* dlh) + { + // assert(not wsrep::thread_service_v1_probe(dlh)); + wsrep::thread_service_v1_deinit(dlh); + } + + static int init_tls_service(void* dlh, + wsrep::tls_service* tls_service) + { + assert(tls_service); + if (not wsrep::tls_service_v1_probe(dlh)) + { + return wsrep::tls_service_v1_init(dlh, tls_service); + } + return 1; + } + + static void deinit_tls_service(void* dlh) + { + // assert(not wsrep::tls_service_v1_probe(dlh)); + wsrep::tls_service_v1_deinit(dlh); + } +} + +void wsrep::wsrep_provider_v26::init_services( + const wsrep::provider::services& services) +{ + if (services.thread_service) + { + if (init_thread_service(wsrep_->dlh, services.thread_service)) + { + throw wsrep::runtime_error("Failed to initialize thread service"); + } + services_enabled_.thread_service = services.thread_service; + } + if (services.tls_service) + { + if (init_tls_service(wsrep_->dlh, services.tls_service)) + { + throw wsrep::runtime_error("Failed to initialze TLS service"); + } + services_enabled_.tls_service = services.tls_service; + } +} + +void wsrep::wsrep_provider_v26::deinit_services() +{ + if (services_enabled_.tls_service) + deinit_tls_service(wsrep_->dlh); + if (services_enabled_.thread_service) + deinit_thread_service(wsrep_->dlh); } wsrep::wsrep_provider_v26::wsrep_provider_v26( @@ -619,6 +673,7 @@ wsrep::wsrep_provider_v26::wsrep_provider_v26( const wsrep::provider::services& services) : provider(server_state) , wsrep_() + , services_enabled_() { wsrep_gtid_t state_id; bool encryption_enabled = server_state.encryption_service() && @@ -653,11 +708,7 @@ wsrep::wsrep_provider_v26::wsrep_provider_v26( throw wsrep::runtime_error("Failed to load wsrep library"); } - if (services.thread_service && - init_thread_service(wsrep_->dlh, services.thread_service)) - { - throw wsrep::runtime_error("Failed to initialize thread service"); - } + init_services(services); if (wsrep_->init(wsrep_, &init_args) != WSREP_OK) { @@ -683,6 +734,8 @@ wsrep::wsrep_provider_v26::wsrep_provider_v26( wsrep::wsrep_provider_v26::~wsrep_provider_v26() { + wsrep_->free(wsrep_); + deinit_services(); wsrep_unload(wsrep_); } diff --git a/src/wsrep_provider_v26.hpp b/src/wsrep_provider_v26.hpp index 47a91ce..1859b67 100644 --- a/src/wsrep_provider_v26.hpp +++ b/src/wsrep_provider_v26.hpp @@ -30,7 +30,8 @@ namespace wsrep class wsrep_provider_v26 : public wsrep::provider { public: - + void init_services(const wsrep::provider::services& services); + void deinit_services(); wsrep_provider_v26(wsrep::server_state&, const std::string&, const std::string&, const wsrep::provider::services& services); @@ -107,6 +108,7 @@ namespace wsrep wsrep_provider_v26(const wsrep_provider_v26&); wsrep_provider_v26& operator=(const wsrep_provider_v26); struct wsrep_st* wsrep_; + services services_enabled_; }; } diff --git a/wsrep-API/v26 b/wsrep-API/v26 index 12a50c4..76cf223 160000 --- a/wsrep-API/v26 +++ b/wsrep-API/v26 @@ -1 +1 @@ -Subproject commit 12a50c43b112648fec3b1213a1470a85aca55f2c +Subproject commit 76cf223c690845bbf561cb820a46e06a18ad80d1