From 9318a50d18cea0c714414bc210f54c132a1886ca Mon Sep 17 00:00:00 2001 From: Teemu Ollakka Date: Fri, 26 Jun 2020 15:28:25 +0300 Subject: [PATCH] Wsrep TLS service This commit defines a TLS service interface. If the implementation is provided by the application when the provider is loaded, appropriate hooks are probed from the provider and the provider side hooks are initialized after the provider is loaded. A sample implementation to demostrate the use of TLS interface is provided in dbsim/db_tls.cpp. Also contains a change to thread service interface: The thread exit virtual method was changed to function pointer to allow thread exit path which does not involve C++. --- dbsim/CMakeLists.txt | 1 + dbsim/db_params.cpp | 5 + dbsim/db_params.hpp | 2 + dbsim/db_simulator.cpp | 7 + dbsim/db_threads.cpp | 67 +++-- dbsim/db_threads.hpp | 3 +- dbsim/db_tls.cpp | 451 +++++++++++++++++++++++++++++++ dbsim/db_tls.hpp | 62 +++++ include/wsrep/provider.hpp | 4 + include/wsrep/thread_service.hpp | 25 +- include/wsrep/tls_service.hpp | 107 ++++++++ src/CMakeLists.txt | 1 + src/service_helpers.hpp | 106 ++++++++ src/thread_service_v1.cpp | 66 ++--- src/thread_service_v1.hpp | 8 + src/tls_service_v1.cpp | 232 ++++++++++++++++ src/tls_service_v1.hpp | 54 ++++ src/wsrep_provider_v26.cpp | 65 ++++- src/wsrep_provider_v26.hpp | 4 +- wsrep-API/v26 | 2 +- 20 files changed, 1193 insertions(+), 79 deletions(-) create mode 100644 dbsim/db_tls.cpp create mode 100644 dbsim/db_tls.hpp create mode 100644 include/wsrep/tls_service.hpp create mode 100644 src/service_helpers.hpp create mode 100644 src/tls_service_v1.cpp create mode 100644 src/tls_service_v1.hpp diff --git a/dbsim/CMakeLists.txt b/dbsim/CMakeLists.txt index 012200b..aee964c 100644 --- a/dbsim/CMakeLists.txt +++ b/dbsim/CMakeLists.txt @@ -13,6 +13,7 @@ add_executable(dbsim db_simulator.cpp db_storage_engine.cpp db_threads.cpp + db_tls.cpp dbsim.cpp ) diff --git a/dbsim/db_params.cpp b/dbsim/db_params.cpp index 3524eeb..4fbefd0 100644 --- a/dbsim/db_params.cpp +++ b/dbsim/db_params.cpp @@ -87,6 +87,11 @@ db::params db::parse_args(int argc, char** argv) po::value(¶ms.cond_checks), "Enable checks for correct condition variable use. " " Effective only if thread-instrumentation is enabled") + ("tls-service", + po::value(¶ms.tls_service), + "Configure TLS service stubs.\n0 default disabled\n1 enabled\n" + "2 enabled with short read/write and renegotiation simulation\n" + "3 enabled with error simulation.") ; try { diff --git a/dbsim/db_params.hpp b/dbsim/db_params.hpp index c97f4d3..6443e13 100644 --- a/dbsim/db_params.hpp +++ b/dbsim/db_params.hpp @@ -42,6 +42,7 @@ namespace db int fast_exit; int thread_instrumentation; bool cond_checks; + int tls_service; params() : n_servers(0) , n_clients(0) @@ -58,6 +59,7 @@ namespace db , fast_exit(0) , thread_instrumentation() , cond_checks() + , tls_service() { } }; diff --git a/dbsim/db_simulator.cpp b/dbsim/db_simulator.cpp index 9b639b7..0971b11 100644 --- a/dbsim/db_simulator.cpp +++ b/dbsim/db_simulator.cpp @@ -20,6 +20,7 @@ #include "db_simulator.hpp" #include "db_client.hpp" #include "db_threads.hpp" +#include "db_tls.hpp" #include "wsrep/logger.hpp" @@ -27,6 +28,7 @@ #include static db::ti thread_instrumentation; +static db::tls tls_service; void db::simulator::run() { @@ -36,6 +38,7 @@ void db::simulator::run() std::cout << "Results:\n"; std::cout << stats() << std::endl; std::cout << db::ti::stats() << std::endl; + std::cout << db::tls::stats() << std::endl; } void db::simulator::sst(db::server& server, @@ -114,6 +117,7 @@ void db::simulator::start() { thread_instrumentation.level(params_.thread_instrumentation); thread_instrumentation.cond_checks(params_.cond_checks); + tls_service.init(params_.tls_service); wsrep::log_info() << "Provider: " << params_.wsrep_provider; std::string cluster_address(build_cluster_address()); @@ -149,6 +153,9 @@ void db::simulator::start() services.thread_service = params_.thread_instrumentation ? &thread_instrumentation : nullptr; + services.tls_service = params_.tls_service + ? &tls_service + : nullptr; if (server.server_state().load_provider(params_.wsrep_provider, server_options, services)) { diff --git a/dbsim/db_threads.cpp b/dbsim/db_threads.cpp index a2166a2..d066c65 100644 --- a/dbsim/db_threads.cpp +++ b/dbsim/db_threads.cpp @@ -35,6 +35,7 @@ #include #include +extern "C" { static void* start_thread(void* args_ptr); } namespace { struct ti_obj @@ -186,7 +187,6 @@ namespace } }; - void* thread_start_fn(void* args_ptr); class ti_thread : public ti_obj { @@ -209,7 +209,7 @@ namespace int run(void* (*fn)(void *), void* args) { auto ta(new thread_args{this, fn, args}); - return pthread_create(&th_, nullptr, thread_start_fn, ta); + return pthread_create(&th_, nullptr, start_thread, ta); } int detach() @@ -254,24 +254,6 @@ namespace bool detached_; }; - void* thread_start_fn(void* args_ptr) - { - thread_args* ta(reinterpret_cast(args_ptr)); - ti_thread* thread = reinterpret_cast(ta->this_thread); - pthread_setspecific(this_thread_key, thread); - void* (*fn)(void*) = ta->fn; - void* args = ta->args; - delete ta; - void* ret((*fn)(args)); - pthread_setspecific(this_thread_key, nullptr); - - // If we end here the thread returned instead of calling - // pthread_exit() - if (thread->detached()) - delete thread; - return ret; - } - class ti_mutex : public ti_obj { public: @@ -482,6 +464,42 @@ int db::ti::after_init() // Thread // ////////////////////////////////////////////////////////////////////////////// +extern "C" +{ +static void* start_thread(void* args_ptr) +{ + thread_args* ta(reinterpret_cast(args_ptr)); + ti_thread* thread = reinterpret_cast(ta->this_thread); + pthread_setspecific(this_thread_key, thread); + void* (*fn)(void*) = ta->fn; + void* args = ta->args; + delete ta; + void* ret = (*fn)(args); + pthread_setspecific(this_thread_key, nullptr); + // If we end here the thread returned instead of calling + // pthread_exit() + if (thread->detached()) + delete thread; + return ret; +} + +WSREP_NORETURN +static void exit_thread(wsrep::thread_service::thread* thread, void* retval) +{ + pthread_setspecific(this_thread_key, nullptr); + ti_thread* th(reinterpret_cast(thread)); + th->retval(retval); + if (th->detached()) + delete th; + pthread_exit(retval); +} +} // extern "C" + +db::ti::ti() +{ + thread_service::exit = exit_thread; +} + const wsrep::thread_service::thread_key* db::ti::create_thread_key(const char* name) WSREP_NOEXCEPT { @@ -513,15 +531,6 @@ int db::ti::detach(wsrep::thread_service::thread* thread) WSREP_NOEXCEPT return reinterpret_cast(thread)->detach(); } -void db::ti::exit(wsrep::thread_service::thread* thread, void* retval) WSREP_NOEXCEPT -{ - ti_thread* th(reinterpret_cast(thread)); - th->retval(retval); - if (th->detached()) - delete th; - pthread_exit(retval); -} - int db::ti::equal(wsrep::thread_service::thread* thread_1, wsrep::thread_service::thread* thread_2) WSREP_NOEXCEPT { diff --git a/dbsim/db_threads.hpp b/dbsim/db_threads.hpp index 01ad956..32d44c1 100644 --- a/dbsim/db_threads.hpp +++ b/dbsim/db_threads.hpp @@ -28,7 +28,7 @@ namespace db class ti : public wsrep::thread_service { public: - ti() { } + ti(); int before_init() override; int after_init() override; @@ -41,7 +41,6 @@ namespace db int detach(wsrep::thread_service::thread*) WSREP_NOEXCEPT override; int equal(wsrep::thread_service::thread*, wsrep::thread_service::thread*) WSREP_NOEXCEPT override; - void exit(wsrep::thread_service::thread*, void*) WSREP_NOEXCEPT override; int join(wsrep::thread_service::thread*, void**) WSREP_NOEXCEPT override; wsrep::thread_service::thread* self() WSREP_NOEXCEPT override; int setschedparam(wsrep::thread_service::thread*, int, diff --git a/dbsim/db_tls.cpp b/dbsim/db_tls.cpp new file mode 100644 index 0000000..c242668 --- /dev/null +++ b/dbsim/db_tls.cpp @@ -0,0 +1,451 @@ +/* + * Copyright (C) 2020 Codership Oy + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see . + */ + +/** @file db_tls.cpp + * + * This file demonstrates the use of TLS service. It does not implement + * real encryption, but may manipulate stream bytes for testing purposes. + */ + +#include "db_tls.hpp" + +#include "wsrep/logger.hpp" + +#include // read() +#include +#include +#include // send() +#include +#include +#include + +#include +#include + +namespace +{ + class db_stream : public wsrep::tls_stream + { + public: + db_stream(int fd, int mode) + : fd_(fd) + , state_(s_initialized) + , last_error_() + , mode_(mode) + , stats_() + , is_blocking_() + { + int val(fcntl(fd_, F_GETFL, 0)); + is_blocking_ = not (val & O_NONBLOCK); + } + struct stats + { + size_t bytes_read{0}; + size_t bytes_written{0}; + }; + + /* + * in idle --| + * |-> ch -| ^ | -> want_read --| + * |-> sh -| ---- |--> | -> want_write --| + * |----------------------| + */ + enum state + { + s_initialized, + s_client_handshake, + s_server_handshake, + s_idle, + s_want_read, + s_want_write + }; + + int get_error_number() const { return last_error_; } + const void* get_error_category() const + { + return reinterpret_cast(1); + } + + static char* get_error_message(int value, const void*) + { + return ::strerror(value); + } + + enum wsrep::tls_service::status client_handshake(); + + enum wsrep::tls_service::status server_handshake(); + + wsrep::tls_service::op_result read(void*, size_t); + + wsrep::tls_service::op_result write(const void*, size_t); + + enum state state() const { return state_; } + + int fd() const { return fd_; } + void inc_reads(size_t val) { stats_.bytes_read += val; } + void inc_writes(size_t val) { stats_.bytes_written += val; } + const stats& get_stats() const { return stats_; } + private: + enum wsrep::tls_service::status handle_handshake_read(const char* expect); + size_t determine_read_count(size_t max_count) + { + if (is_blocking_ || mode_ < 2) return max_count; + else if (::rand() % 100 == 0) return std::min(size_t(42), max_count); + else return max_count; + } + size_t determine_write_count(size_t count) + { + if (is_blocking_ || mode_ < 2) return count; + else if (::rand() % 100 == 0) return std::min(size_t(43), count); + else return count; + } + + ssize_t do_read(void* buf, size_t max_count) + { + if (is_blocking_ || mode_ < 3 ) + return ::read(fd_, buf, max_count); + else if (::rand() % 1000 == 0) return EINTR; + else return ::read(fd_, buf, max_count); + } + + ssize_t do_write(const void* buf, size_t count) + { + if (is_blocking_ || mode_ < 3) + return ::send(fd_, buf, count, MSG_NOSIGNAL); + else if (::rand() % 1000 == 0) return EINTR; + else return ::send(fd_, buf, count, MSG_NOSIGNAL); + } + + wsrep::tls_service::op_result map_success(ssize_t result) + { + if (is_blocking_ || mode_ < 2) + { + return wsrep::tls_service::op_result{ + wsrep::tls_service::success, size_t(result)}; + } + else if (::rand() % 1000 == 0) + { + wsrep::log_info() << "Success want extra read"; + state_ = s_want_read; + return wsrep::tls_service::op_result{ + wsrep::tls_service::want_read, size_t(result)}; + } + else if (::rand() % 1000 == 0) + { + wsrep::log_info() << "Success want extra write"; + state_ = s_want_write; + return wsrep::tls_service::op_result{ + wsrep::tls_service::want_write, size_t(result)}; + } + else + { + return wsrep::tls_service::op_result{ + wsrep::tls_service::success, size_t(result)}; + } + } + + wsrep::tls_service::op_result map_result(ssize_t result) + { + if (result > 0) + { + return map_success(result); + } + else if (result == 0) + { + return wsrep::tls_service::op_result{ + wsrep::tls_service::eof, 0}; + } + else if (errno == EAGAIN || errno == EWOULDBLOCK) + { + return wsrep::tls_service::op_result{ + wsrep::tls_service::want_read, 0}; + } + else + { + last_error_ = errno; + return wsrep::tls_service::op_result{ + wsrep::tls_service::error, 0}; + } + } + + void clear_error() { last_error_ = 0; } + + int fd_; + enum state state_; + int last_error_; + // Operation mode: + // 1 - simulate handshake exchange + // 2 - simulate errors and short reads + int mode_; + stats stats_; + bool is_blocking_; + }; + + enum wsrep::tls_service::status db_stream::client_handshake() + { + clear_error(); + enum wsrep::tls_service::status ret; + assert(state_ == s_initialized || + state_ == s_client_handshake || + state_ == s_want_write); + if (state_ == s_initialized) + { + (void)::send(fd_, "clie", 4, MSG_NOSIGNAL); + ret = wsrep::tls_service::want_read; + state_ = s_client_handshake; + wsrep::log_info() << this << " client handshake sent"; + stats_.bytes_written += 4; + if (not is_blocking_) return ret; + } + + if (state_ == s_client_handshake) + { + if ((ret = handle_handshake_read("serv")) == + wsrep::tls_service::success) + { + state_ = s_want_write; + ret = wsrep::tls_service::want_write; + } + if (not is_blocking_) return ret; + } + + if (state_ == s_want_write) + { + state_ = s_idle; + ret = wsrep::tls_service::success; + if (not is_blocking_) return ret; + } + + if (not is_blocking_) + { + last_error_ = EPROTO; + ret = wsrep::tls_service::error; + } + return ret; + } + + + + enum wsrep::tls_service::status db_stream::server_handshake() + { + enum wsrep::tls_service::status ret; + assert(state_ == s_initialized || + state_ == s_server_handshake || + state_ == s_want_write); + + if (state_ == s_initialized) + { + ::send(fd_, "serv", 4, MSG_NOSIGNAL); + ret = wsrep::tls_service::want_read; + state_ = s_server_handshake; + stats_.bytes_written += 4; + if (not is_blocking_) return ret; + } + + if (state_ == s_server_handshake) + { + if ((ret = handle_handshake_read("clie")) == + wsrep::tls_service::success) + { + state_ = s_want_write; + ret = wsrep::tls_service::want_write; + } + if (not is_blocking_) return ret; + } + + if (state_ == s_want_write) + { + state_ = s_idle; + ret = wsrep::tls_service::success; + if (not is_blocking_) return ret; + } + + if (not is_blocking_) + { + last_error_ = EPROTO; + ret = wsrep::tls_service::error; + } + return ret; + } + + enum wsrep::tls_service::status db_stream::handle_handshake_read( + const char* expect) + { + assert(::strlen(expect) >= 4); + char buf[4] = { }; + ssize_t read_result(::read(fd_, buf, sizeof(buf))); + if (read_result > 0) stats_.bytes_read += size_t(read_result); + enum wsrep::tls_service::status ret; + if (read_result == -1 && + (errno == EWOULDBLOCK || errno == EAGAIN)) + { + ret = wsrep::tls_service::want_read; + } + else if (read_result == 0) + { + ret = wsrep::tls_service::eof; + } + else if (read_result != 4 || ::memcmp(buf, expect, 4)) + { + last_error_ = EPROTO; + ret = wsrep::tls_service::error; + } + else + { + wsrep::log_info() << "Handshake success: " << std::string(buf, 4); + ret = wsrep::tls_service::success; + } + return ret; + } + + wsrep::tls_service::op_result db_stream::read(void* buf, size_t max_count) + { + clear_error(); + if (state_ == s_want_read) + { + state_ = s_idle; + if (max_count == 0) + return wsrep::tls_service::op_result{ + wsrep::tls_service::success, 0}; + } + max_count = determine_read_count(max_count); + ssize_t read_result(do_read(buf, max_count)); + if (read_result > 0) + { + inc_reads(size_t(read_result)); + } + return map_result(read_result); + } + + wsrep::tls_service::op_result db_stream::write( + const void* buf, size_t count) + { + clear_error(); + if (state_ == s_want_write) + { + state_ = s_idle; + if (count == 0) + return wsrep::tls_service::op_result{ + wsrep::tls_service::success, 0}; + } + count = determine_write_count(count); + ssize_t write_result(do_write(buf, count)); + if (write_result > 0) + { + inc_writes(size_t(write_result)); + } + return map_result(write_result); + } +} + + +static db_stream::stats global_stats; +std::mutex global_stats_lock; +static int global_mode; + +static void merge_to_global_stats(const db_stream::stats& stats) +{ + std::lock_guard lock(global_stats_lock); + global_stats.bytes_read += stats.bytes_read; + global_stats.bytes_written += stats.bytes_written; +} + +wsrep::tls_stream* db::tls::create_tls_stream(int fd) WSREP_NOEXCEPT +{ + auto ret(new db_stream(fd, global_mode)); + wsrep::log_debug() << "New DB stream: " << ret; + return ret; +} + +void db::tls::destroy(wsrep::tls_stream* stream) WSREP_NOEXCEPT +{ + auto dbs(static_cast(stream)); + merge_to_global_stats(dbs->get_stats()); + wsrep::log_debug() << "Stream destroy: " << dbs->get_stats().bytes_read + << " " << dbs->get_stats().bytes_written; + wsrep::log_debug() << "Stream destroy" << dbs; + delete dbs; +} + +int db::tls::get_error_number(const wsrep::tls_stream* stream) + const WSREP_NOEXCEPT +{ + return static_cast(stream)->get_error_number(); +} + +const void* db::tls::get_error_category(const wsrep::tls_stream* stream) + const WSREP_NOEXCEPT +{ + return static_cast(stream)->get_error_category(); +} + +const char* db::tls::get_error_message(const wsrep::tls_stream*, + int value, const void* category) + const WSREP_NOEXCEPT +{ + return db_stream::get_error_message(value, category); +} + +enum wsrep::tls_service::status +db::tls::client_handshake(wsrep::tls_stream* stream) WSREP_NOEXCEPT +{ + return static_cast(stream)->client_handshake(); +} + +enum wsrep::tls_service::status +db::tls::server_handshake(wsrep::tls_stream* stream) WSREP_NOEXCEPT +{ + return static_cast(stream)->server_handshake(); +} + +wsrep::tls_service::op_result db::tls::read( + wsrep::tls_stream* stream, + void* buf, size_t max_count) WSREP_NOEXCEPT +{ + return static_cast(stream)->read(buf, max_count); +} + +wsrep::tls_service::op_result db::tls::write( + wsrep::tls_stream* stream, + const void* buf, size_t count) WSREP_NOEXCEPT +{ + return static_cast(stream)->write(buf, count); +} + +wsrep::tls_service::status +db::tls::shutdown(wsrep::tls_stream*) WSREP_NOEXCEPT +{ + // @todo error simulation + return wsrep::tls_service::success; +} + + +void db::tls::init(int mode) +{ + global_mode = mode; +} + +std::string db::tls::stats() +{ + std::ostringstream oss; + oss << "Transport stats:\n" + << " bytes_read: " << global_stats.bytes_read << "\n" + << " bytes_written: " << global_stats.bytes_written << "\n"; + return oss.str(); +} diff --git a/dbsim/db_tls.hpp b/dbsim/db_tls.hpp new file mode 100644 index 0000000..97c4387 --- /dev/null +++ b/dbsim/db_tls.hpp @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2020 Codership Oy + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see . + */ + +#ifndef WSREP_DB_TLS_HPP +#define WSREP_DB_TLS_HPP + +#include "wsrep/tls_service.hpp" + +#include + +namespace db +{ + class tls : public wsrep::tls_service + { + public: + virtual wsrep::tls_stream* create_tls_stream(int) + WSREP_NOEXCEPT override; + virtual void destroy(wsrep::tls_stream*) WSREP_NOEXCEPT override; + virtual int get_error_number(const wsrep::tls_stream*) const + WSREP_NOEXCEPT override; + virtual const void* get_error_category(const wsrep::tls_stream*) const + WSREP_NOEXCEPT override; + virtual const char* get_error_message(const wsrep::tls_stream*, + int, const void*) const + WSREP_NOEXCEPT override; + virtual enum wsrep::tls_service::status + client_handshake(wsrep::tls_stream*) + WSREP_NOEXCEPT override; + virtual enum wsrep::tls_service::status + server_handshake(wsrep::tls_stream*) + WSREP_NOEXCEPT override; + virtual wsrep::tls_service::op_result + read(wsrep::tls_stream*, void* buf, size_t max_count) + WSREP_NOEXCEPT override; + virtual wsrep::tls_service::op_result + write(wsrep::tls_stream*, const void* buf, size_t count) + WSREP_NOEXCEPT override; + virtual wsrep::tls_service::status + shutdown(wsrep::tls_stream*) WSREP_NOEXCEPT override; + + static void init(int mode); + static std::string stats(); + }; +}; + +#endif // WSREP_DB_TLS_HPP diff --git a/include/wsrep/provider.hpp b/include/wsrep/provider.hpp index a828a6b..4c85880 100644 --- a/include/wsrep/provider.hpp +++ b/include/wsrep/provider.hpp @@ -45,6 +45,8 @@ namespace wsrep class server_state; class high_priority_service; class thread_service; + class tls_service; + class stid { public: @@ -419,8 +421,10 @@ namespace wsrep struct services { wsrep::thread_service* thread_service; + wsrep::tls_service* tls_service; services() : thread_service() + , tls_service() { } }; diff --git a/include/wsrep/thread_service.hpp b/include/wsrep/thread_service.hpp index 1f8d2f0..702d71c 100644 --- a/include/wsrep/thread_service.hpp +++ b/include/wsrep/thread_service.hpp @@ -28,6 +28,9 @@ * condition variables. */ +#ifndef WSREP_THREAD_SERVICE_HPP +#define WSREP_THREAD_SERVICE_HPP + #include // size_t #include "compiler.hpp" @@ -41,13 +44,14 @@ namespace wsrep { public: + thread_service() : exit() { } virtual ~thread_service() { } - class thread_key { }; - class thread { }; - class mutex_key { }; - class mutex { }; - class cond_key { }; - class cond { }; + struct thread_key { }; + struct thread { }; + struct mutex_key { }; + struct mutex { }; + struct cond_key { }; + struct cond { }; /** * Method will be called before library side thread @@ -69,7 +73,12 @@ namespace wsrep = 0; virtual int detach(thread*) WSREP_NOEXCEPT = 0; virtual int equal(thread*, thread*) WSREP_NOEXCEPT = 0; - WSREP_NORETURN virtual void exit(thread*, void* retval) WSREP_NOEXCEPT = 0; + + /* + * This unlike others is a function pointer to + * avoid having C++ methods on thread exit codepath. + */ + WSREP_NORETURN void (*exit)(thread*, void* retval); virtual int join(thread*, void** retval) WSREP_NOEXCEPT = 0; virtual thread* self() WSREP_NOEXCEPT = 0; virtual int setschedparam(thread*, int, @@ -98,3 +107,5 @@ namespace wsrep virtual int broadcast(cond*) WSREP_NOEXCEPT = 0; }; } // namespace wsrep + +#endif // WSREP_THREAD_SERVICE_HPP diff --git a/include/wsrep/tls_service.hpp b/include/wsrep/tls_service.hpp new file mode 100644 index 0000000..07d2064 --- /dev/null +++ b/include/wsrep/tls_service.hpp @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2020 Codership Oy + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see . + */ + + +/** @file tls_service.hpp + * + * Service interface for interacting with DBMS provided + * TLS and encryption facilities. + */ + +#ifndef WSREP_TLS_SERVICE_HPP +#define WSREP_TLS_SERVICE_HPP + +#include "compiler.hpp" + +#include // ssize_t + +namespace wsrep +{ + + /* Type tags for TLS context and TLS stream. */ + struct tls_context { }; + struct tls_stream { }; + + /** @class tls_service + * + * TLS service interface. This provides an interface corresponding + * to wsrep-API TLS service. For details see wsrep-API/wsrep_tls_service.h + */ + class tls_service + { + public: + enum status + { + success = 0, + want_read, + want_write, + eof, + error + }; + + struct op_result + { + /** Status code of the operation of negative system error number. */ + ssize_t status; + /** Bytes transferred from/to given buffer during the operation. */ + size_t bytes_transferred; + }; + + virtual ~tls_service() { } + /** + * @return Zero on success, system error code on failure. + */ + virtual tls_stream* create_tls_stream(int fd) WSREP_NOEXCEPT = 0; + virtual void destroy(tls_stream*) WSREP_NOEXCEPT = 0; + + virtual int get_error_number(const tls_stream*) const WSREP_NOEXCEPT = 0; + virtual const void* get_error_category(const tls_stream*) const WSREP_NOEXCEPT = 0; + virtual const char* get_error_message(const tls_stream*, + int value, const void* category) + const WSREP_NOEXCEPT = 0; + /** + * @return Status enum. + */ + virtual status client_handshake(tls_stream*) WSREP_NOEXCEPT = 0; + + /** + * @return Status enum or negative error code. + */ + virtual status server_handshake(tls_stream*) WSREP_NOEXCEPT = 0; + + /** + * Read at most max_count bytes into buf. + */ + virtual op_result read(tls_stream*, + void* buf, size_t max_count) WSREP_NOEXCEPT = 0; + + /** + * Write at most count bytes from buf. + */ + virtual op_result write(tls_stream*, + const void* buf, size_t count) WSREP_NOEXCEPT = 0; + + /** + * Shutdown TLS stream. + */ + virtual status shutdown(tls_stream*) WSREP_NOEXCEPT = 0; + }; +} + +#endif // WSREP_TLS_SERVICE_HPP diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1a49f95..0401494 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,6 +16,7 @@ add_library(wsrep-lib server_state.cpp thread.cpp thread_service_v1.cpp + tls_service_v1.cpp transaction.cpp uuid.cpp wsrep_provider_v26.cpp) diff --git a/src/service_helpers.hpp b/src/service_helpers.hpp new file mode 100644 index 0000000..6e9f9ca --- /dev/null +++ b/src/service_helpers.hpp @@ -0,0 +1,106 @@ +/* + * Copyright (C) 2020 Codership Oy + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see . + */ + +#ifndef WSREP_SERVICE_HELPERS_HPP +#define WSREP_SERVICE_HELPERS_HPP + +#include "wsrep/logger.hpp" + +#include +#include + +namespace wsrep_impl +{ + template + int service_probe(void* dlh, const char* symbol, const char* service_name) + { + union { + InitFn dlfun; + void* obj; + } alias; + // Clear previous errors + (void)dlerror(); + alias.obj = dlsym(dlh, symbol); + if (alias.obj) + { + return 0; + } + else + { + wsrep::log_warning() << "Support for " << service_name + << " " << symbol + << " not found from provider: " + << dlerror(); + return ENOTSUP; + } + } + + + template + int service_init(void* dlh, + const char* symbol, + ServiceCallbacks service_callbacks, + const char* service_name) + { + union { + InitFn dlfun; + void* obj; + } alias; + // Clear previous errors + (void)dlerror(); + alias.obj = dlsym(dlh, symbol); + if (alias.obj) + { + wsrep::log_info() << "Initializing " << service_name; + return (*alias.dlfun)(service_callbacks); + } + else + { + wsrep::log_info() + << "Provider does not support " << service_name; + return ENOTSUP; + } + } + + template + void service_deinit(void* dlh, const char* symbol, const char* service_name) + { + union { + DeinitFn dlfun; + void* obj; + } alias; + // Clear previous errors + (void)dlerror(); + alias.obj = dlsym(dlh, symbol); + if (alias.obj) + { + wsrep::log_info() << "Deinitializing " << service_name; + (*alias.dlfun)(); + } + else + { + wsrep::log_info() + << "Provider does not support deinitializing of " + << service_name; + } + } +} + +#endif // WSREP_SERVICE_HELPERS_HPP + diff --git a/src/thread_service_v1.cpp b/src/thread_service_v1.cpp index 1d07af2..ee81ba3 100644 --- a/src/thread_service_v1.cpp +++ b/src/thread_service_v1.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2019 Codership Oy + * Copyright (C) 2019-2020 Codership Oy * * This file is part of wsrep-lib. * @@ -18,6 +18,7 @@ */ #include "thread_service_v1.hpp" +#include "service_helpers.hpp" #include "wsrep/thread_service.hpp" #include "wsrep/logger.hpp" @@ -36,6 +37,7 @@ namespace wsrep_thread_service_v1 // Pointer to thread service implementation provided by // the application. static wsrep::thread_service* thread_service_impl{ 0 }; + static std::atomic use_count; static const wsrep_thread_key_t* thread_key_create_cb(const char* name) { @@ -236,50 +238,48 @@ namespace wsrep_thread_service_v1 int wsrep::thread_service_v1_probe(void* dlh) { - typedef int (*init_fn)(wsrep_thread_service_v1_t*); - union { - init_fn dlfun; - void* obj; - } alias; - // Clear previous errors - (void)dlerror(); - alias.obj = dlsym(dlh, WSREP_THREAD_SERVICE_INIT_FUNC); - if (alias.obj) + typedef int (*init_fn)(wsrep_thread_service_v1_t*); + typedef void (*deinit_fn)(); + if (wsrep_impl::service_probe( + dlh, WSREP_THREAD_SERVICE_INIT_FUNC_V1, "thread service v1") || + wsrep_impl::service_probe( + dlh, WSREP_THREAD_SERVICE_DEINIT_FUNC_V1, "thread service v1")) { - wsrep::log_info() - << "Found support for thread service v1 from provider"; - return 0; + wsrep::log_warning() << "Provider does not support thread service v1"; + return 1; } - else - { - wsrep::log_info() << "Thread service v1 not found from provider: " - << dlerror(); - return ENOTSUP; - } - + return 0; } int wsrep::thread_service_v1_init(void* dlh, wsrep::thread_service* thread_service) { if (not (dlh && thread_service)) return EINVAL; - typedef int (*init_fn)(wsrep_thread_service_v1_t*); - union { - init_fn dlfun; - void* obj; - } alias; - alias.obj = dlsym(dlh, WSREP_THREAD_SERVICE_INIT_FUNC); - if (alias.obj) + wsrep_thread_service_v1::thread_service_impl = thread_service; + int ret(0); + if ((ret = wsrep_impl::service_init( + dlh, WSREP_THREAD_SERVICE_INIT_FUNC_V1, + &wsrep_thread_service_v1::thread_service_callbacks, + "thread service v1"))) { - wsrep::log_info() << "Initializing process instrumentation"; - wsrep_thread_service_v1::thread_service_impl = thread_service; - return (*alias.dlfun)(&wsrep_thread_service_v1::thread_service_callbacks); + wsrep_thread_service_v1::thread_service_impl = 0; } else { - wsrep::log_info() - << "Provider does not support process instrumentation"; - return ENOTSUP; + ++wsrep_thread_service_v1::use_count; + } + return ret; +} + +void wsrep::thread_service_v1_deinit(void* dlh) +{ + typedef int (*deinit_fn)(); + wsrep_impl::service_deinit( + dlh, WSREP_THREAD_SERVICE_DEINIT_FUNC_V1, "thread service v1"); + --wsrep_thread_service_v1::use_count; + if (wsrep_thread_service_v1::use_count == 0) + { + wsrep_thread_service_v1::thread_service_impl = 0; } } diff --git a/src/thread_service_v1.hpp b/src/thread_service_v1.hpp index 6fd8b25..b830004 100644 --- a/src/thread_service_v1.hpp +++ b/src/thread_service_v1.hpp @@ -42,6 +42,14 @@ namespace wsrep */ int thread_service_v1_init(void* dlh, wsrep::thread_service* thread_service); + + /** + * Deinitialize the thread service. + * + * @params dlh Handler returned by dlopen(). + */ + void thread_service_v1_deinit(void* dlh); + } #endif // WSREP_THREAD_SERVICE_V1_HPP diff --git a/src/tls_service_v1.cpp b/src/tls_service_v1.cpp new file mode 100644 index 0000000..8746a76 --- /dev/null +++ b/src/tls_service_v1.cpp @@ -0,0 +1,232 @@ +/* + * Copyright (C) 2020 Codership Oy + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see . + */ + +#include "tls_service_v1.hpp" + +#include "wsrep/tls_service.hpp" +#include "wsrep/logger.hpp" +#include "v26/wsrep_tls_service.h" +#include "service_helpers.hpp" + +#include + +namespace wsrep_tls_service_v1 +{ + static wsrep::tls_service* tls_service_impl{0}; + static std::atomic use_count; + + static int tls_stream_init_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + stream->opaque = + tls_service_impl->create_tls_stream(stream->fd); + if (not stream->opaque) + { + return ENOMEM; + } + return 0; + } + + static void tls_stream_deinit_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + tls_service_impl->destroy( + reinterpret_cast(stream->opaque)); + } + + static int tls_stream_get_error_number_cb( + wsrep_tls_context_t*, + const wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + return tls_service_impl->get_error_number( + reinterpret_cast(stream->opaque)); + } + + static const void* tls_stream_get_error_category_cb( + wsrep_tls_context_t*, + const wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + return tls_service_impl->get_error_category( + reinterpret_cast(stream->opaque)); + } + + static const char* tls_error_message_get_cb( + wsrep_tls_context_t*, + const wsrep_tls_stream_t* stream, + int value, const void* category) + { + assert(tls_service_impl); + return tls_service_impl->get_error_message( + reinterpret_cast(stream->opaque), value, category); + } + + static enum wsrep_tls_result map_return_value(ssize_t status) + { + switch (status) + { + case wsrep::tls_service::success: + return wsrep_tls_result_success; + case wsrep::tls_service::want_read: + return wsrep_tls_result_want_read; + case wsrep::tls_service::want_write: + return wsrep_tls_result_want_write; + case wsrep::tls_service::eof: + return wsrep_tls_result_eof; + case wsrep::tls_service::error: + return wsrep_tls_result_error; + default: + assert(status < 0); + return wsrep_tls_result_error; + } + } + + + static enum wsrep_tls_result + tls_stream_client_handshake_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + return map_return_value( + tls_service_impl->client_handshake( + reinterpret_cast(stream->opaque))); + } + + static enum wsrep_tls_result + tls_stream_server_handshake_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + return map_return_value( + tls_service_impl->server_handshake( + reinterpret_cast(stream->opaque))); + } + + static enum wsrep_tls_result tls_stream_read_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream, + void* buf, + size_t max_count, + size_t* bytes_transferred) + { + assert(tls_service_impl); + auto result(tls_service_impl->read( + reinterpret_cast(stream->opaque), + buf, max_count)); + *bytes_transferred = result.bytes_transferred; + return map_return_value(result.status); + } + + static enum wsrep_tls_result tls_stream_write_cb( + wsrep_tls_context_t*, + wsrep_tls_stream_t* stream, + const void* buf, + size_t count, + size_t* bytes_transferred) + { + assert(tls_service_impl); + auto result(tls_service_impl->write( + reinterpret_cast(stream->opaque), + buf, count)); + *bytes_transferred = result.bytes_transferred; + return map_return_value(result.status); + } + + static enum wsrep_tls_result + tls_stream_shutdown_cb(wsrep_tls_context_t*, + wsrep_tls_stream_t* stream) + { + assert(tls_service_impl); + // @todo Handle other values than success. + return map_return_value( + tls_service_impl->shutdown( + reinterpret_cast(stream->opaque))); + } + + static wsrep_tls_service_v1_t tls_service_callbacks = + { + tls_stream_init_cb, + tls_stream_deinit_cb, + tls_stream_get_error_number_cb, + tls_stream_get_error_category_cb, + tls_stream_client_handshake_cb, + tls_stream_server_handshake_cb, + tls_stream_read_cb, + tls_stream_write_cb, + tls_stream_shutdown_cb, + tls_error_message_get_cb, + 0 // we pass NULL context for now. + }; +} + +int wsrep::tls_service_v1_probe(void* dlh) +{ + typedef int (*init_fn)(wsrep_tls_service_v1_t*); + typedef void (*deinit_fn)(); + if (wsrep_impl::service_probe( + dlh, WSREP_TLS_SERVICE_INIT_FUNC_V1, "tls service v1") || + wsrep_impl::service_probe( + dlh, WSREP_TLS_SERVICE_DEINIT_FUNC_V1, "tls service v1")) + { + wsrep::log_warning() << "Provider does not support tls service v1"; + return 1; + } + return 0; +} + +int wsrep::tls_service_v1_init(void* dlh, + wsrep::tls_service* tls_service) +{ + if (not (dlh && tls_service)) return EINVAL; + + typedef int (*init_fn)(wsrep_tls_service_v1_t*); + wsrep_tls_service_v1::tls_service_impl = tls_service; + int ret(0); + if ((ret = wsrep_impl::service_init( + dlh, WSREP_TLS_SERVICE_INIT_FUNC_V1, + &wsrep_tls_service_v1::tls_service_callbacks, + "tls service v1"))) + { + wsrep_tls_service_v1::tls_service_impl = 0; + } + else + { + ++wsrep_tls_service_v1::use_count; + } + return ret; +} + +void wsrep::tls_service_v1_deinit(void* dlh) +{ + typedef int (*deinit_fn)(); + wsrep_impl::service_deinit( + dlh, WSREP_TLS_SERVICE_DEINIT_FUNC_V1, "tls service v1"); + --wsrep_tls_service_v1::use_count; + if (wsrep_tls_service_v1::use_count == 0) + { + wsrep_tls_service_v1::tls_service_impl = 0; + } +} diff --git a/src/tls_service_v1.hpp b/src/tls_service_v1.hpp new file mode 100644 index 0000000..ee9a14b --- /dev/null +++ b/src/tls_service_v1.hpp @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2020 Codership Oy + * + * This file is part of wsrep-lib. + * + * Wsrep-lib is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * Wsrep-lib is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with wsrep-lib. If not, see . + */ + +#ifndef WSREP_TLS_SERVICE_V1_HPP +#define WSREP_TLS_SERVICE_V1_HPP + +namespace wsrep +{ + class tls_service; + /** + * Probe thread_service_v1 support in loaded library. + * + * @param dlh Handle returned by dlopen(). + * + * @return Zero on success, non-zero system error code on failure. + */ + int tls_service_v1_probe(void *dlh); + + /** + * Initialize TLS service. + * + * @param dlh Handle returned by dlopen(). + * @params thread_service Pointer to wsrep::thread_service implementation. + * + * @return Zero on success, non-zero system error code on failure. + */ + int tls_service_v1_init(void* dlh, + wsrep::tls_service* thread_service); + + /** + * Deinitialize TLS service. + * + * @param dlh Handler returned by dlopen(). + */ + void tls_service_v1_deinit(void* dlh); +} + +#endif // WSREP_TLS_SERVICE_V1_HPP diff --git a/src/wsrep_provider_v26.cpp b/src/wsrep_provider_v26.cpp index 769067b..a579dff 100644 --- a/src/wsrep_provider_v26.cpp +++ b/src/wsrep_provider_v26.cpp @@ -26,8 +26,10 @@ #include "wsrep/exception.hpp" #include "wsrep/logger.hpp" #include "wsrep/thread_service.hpp" +#include "wsrep/tls_service.hpp" #include "thread_service_v1.hpp" +#include "tls_service_v1.hpp" #include "v26/wsrep_api.h" @@ -592,7 +594,7 @@ namespace if (wsrep::thread_service_v1_probe(dlh)) { // No support in library. - return 0; + return 1; } else { @@ -610,6 +612,58 @@ namespace } return 0; } + + static void deinit_thread_service(void* dlh) + { + // assert(not wsrep::thread_service_v1_probe(dlh)); + wsrep::thread_service_v1_deinit(dlh); + } + + static int init_tls_service(void* dlh, + wsrep::tls_service* tls_service) + { + assert(tls_service); + if (not wsrep::tls_service_v1_probe(dlh)) + { + return wsrep::tls_service_v1_init(dlh, tls_service); + } + return 1; + } + + static void deinit_tls_service(void* dlh) + { + // assert(not wsrep::tls_service_v1_probe(dlh)); + wsrep::tls_service_v1_deinit(dlh); + } +} + +void wsrep::wsrep_provider_v26::init_services( + const wsrep::provider::services& services) +{ + if (services.thread_service) + { + if (init_thread_service(wsrep_->dlh, services.thread_service)) + { + throw wsrep::runtime_error("Failed to initialize thread service"); + } + services_enabled_.thread_service = services.thread_service; + } + if (services.tls_service) + { + if (init_tls_service(wsrep_->dlh, services.tls_service)) + { + throw wsrep::runtime_error("Failed to initialze TLS service"); + } + services_enabled_.tls_service = services.tls_service; + } +} + +void wsrep::wsrep_provider_v26::deinit_services() +{ + if (services_enabled_.tls_service) + deinit_tls_service(wsrep_->dlh); + if (services_enabled_.thread_service) + deinit_thread_service(wsrep_->dlh); } wsrep::wsrep_provider_v26::wsrep_provider_v26( @@ -619,6 +673,7 @@ wsrep::wsrep_provider_v26::wsrep_provider_v26( const wsrep::provider::services& services) : provider(server_state) , wsrep_() + , services_enabled_() { wsrep_gtid_t state_id; bool encryption_enabled = server_state.encryption_service() && @@ -653,11 +708,7 @@ wsrep::wsrep_provider_v26::wsrep_provider_v26( throw wsrep::runtime_error("Failed to load wsrep library"); } - if (services.thread_service && - init_thread_service(wsrep_->dlh, services.thread_service)) - { - throw wsrep::runtime_error("Failed to initialize thread service"); - } + init_services(services); if (wsrep_->init(wsrep_, &init_args) != WSREP_OK) { @@ -683,6 +734,8 @@ wsrep::wsrep_provider_v26::wsrep_provider_v26( wsrep::wsrep_provider_v26::~wsrep_provider_v26() { + wsrep_->free(wsrep_); + deinit_services(); wsrep_unload(wsrep_); } diff --git a/src/wsrep_provider_v26.hpp b/src/wsrep_provider_v26.hpp index 47a91ce..1859b67 100644 --- a/src/wsrep_provider_v26.hpp +++ b/src/wsrep_provider_v26.hpp @@ -30,7 +30,8 @@ namespace wsrep class wsrep_provider_v26 : public wsrep::provider { public: - + void init_services(const wsrep::provider::services& services); + void deinit_services(); wsrep_provider_v26(wsrep::server_state&, const std::string&, const std::string&, const wsrep::provider::services& services); @@ -107,6 +108,7 @@ namespace wsrep wsrep_provider_v26(const wsrep_provider_v26&); wsrep_provider_v26& operator=(const wsrep_provider_v26); struct wsrep_st* wsrep_; + services services_enabled_; }; } diff --git a/wsrep-API/v26 b/wsrep-API/v26 index 12a50c4..76cf223 160000 --- a/wsrep-API/v26 +++ b/wsrep-API/v26 @@ -1 +1 @@ -Subproject commit 12a50c43b112648fec3b1213a1470a85aca55f2c +Subproject commit 76cf223c690845bbf561cb820a46e06a18ad80d1