Wsrep TLS service

This commit defines a TLS service interface. If the implementation is
provided by the application when the provider is loaded, appropriate
hooks are probed from the provider and the provider side hooks are
initialized after the provider is loaded.

A sample implementation to demostrate the use of TLS interface
is provided in dbsim/db_tls.cpp.

Also contains a change to thread service interface: The
thread exit virtual method was changed to function pointer
to allow thread exit path which does not involve C++.

src/CMakeLists.txt

@@ -16,6 +16,7 @@ add_library(wsrep-lib
   server_state.cpp
   thread.cpp
   thread_service_v1.cpp
+  tls_service_v1.cpp
   transaction.cpp
   uuid.cpp
   wsrep_provider_v26.cpp)

src/service_helpers.hpp

@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2020 Codership Oy <info@codership.com>
+ *
+ * This file is part of wsrep-lib.
+ *
+ * Wsrep-lib is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * Wsrep-lib is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with wsrep-lib.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#ifndef WSREP_SERVICE_HELPERS_HPP
+#define WSREP_SERVICE_HELPERS_HPP
+
+#include "wsrep/logger.hpp"
+
+#include <dlfcn.h>
+#include <cerrno>
+
+namespace wsrep_impl
+{
+    template <typename InitFn>
+    int service_probe(void* dlh, const char* symbol, const char* service_name)
+    {
+        union {
+            InitFn dlfun;
+            void* obj;
+        } alias;
+        // Clear previous errors
+        (void)dlerror();
+        alias.obj = dlsym(dlh, symbol);
+        if (alias.obj)
+        {
+            return 0;
+        }
+        else
+        {
+            wsrep::log_warning() << "Support for " << service_name
+                                 << " " << symbol
+                                 << " not found from provider: "
+                                 << dlerror();
+            return ENOTSUP;
+        }
+    }
+
+
+    template <typename InitFn, typename ServiceCallbacks>
+    int service_init(void* dlh,
+                     const char* symbol,
+                     ServiceCallbacks service_callbacks,
+                     const char* service_name)
+    {
+        union {
+            InitFn dlfun;
+            void* obj;
+        } alias;
+        // Clear previous errors
+        (void)dlerror();
+        alias.obj = dlsym(dlh, symbol);
+        if (alias.obj)
+        {
+            wsrep::log_info() << "Initializing " << service_name;
+            return (*alias.dlfun)(service_callbacks);
+        }
+        else
+        {
+            wsrep::log_info()
+                << "Provider does not support " << service_name;
+            return ENOTSUP;
+        }
+    }
+
+    template <typename DeinitFn>
+    void service_deinit(void* dlh, const char* symbol, const char* service_name)
+    {
+        union {
+            DeinitFn dlfun;
+            void* obj;
+        } alias;
+        // Clear previous errors
+        (void)dlerror();
+        alias.obj = dlsym(dlh, symbol);
+        if (alias.obj)
+        {
+            wsrep::log_info() << "Deinitializing " << service_name;
+            (*alias.dlfun)();
+        }
+        else
+        {
+            wsrep::log_info()
+                << "Provider does not support deinitializing of "
+                << service_name;
+        }
+    }
+}
+
+#endif // WSREP_SERVICE_HELPERS_HPP
+

src/thread_service_v1.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2019 Codership Oy <info@codership.com>
+ * Copyright (C) 2019-2020 Codership Oy <info@codership.com>
  *
  * This file is part of wsrep-lib.
  *
@@ -18,6 +18,7 @@
  */
 
 #include "thread_service_v1.hpp"
+#include "service_helpers.hpp"
 
 #include "wsrep/thread_service.hpp"
 #include "wsrep/logger.hpp"
@@ -36,6 +37,7 @@ namespace wsrep_thread_service_v1
     // Pointer to thread service implementation provided by
     // the application.
     static wsrep::thread_service* thread_service_impl{ 0 };
+    static std::atomic<size_t> use_count;
 
     static const wsrep_thread_key_t* thread_key_create_cb(const char* name)
     {
@@ -236,50 +238,48 @@ namespace wsrep_thread_service_v1
 
 int wsrep::thread_service_v1_probe(void* dlh)
 {
-   typedef int (*init_fn)(wsrep_thread_service_v1_t*);
-    union {
-        init_fn dlfun;
-        void* obj;
-    } alias;
-    // Clear previous errors
-    (void)dlerror();
-    alias.obj = dlsym(dlh, WSREP_THREAD_SERVICE_INIT_FUNC);
-    if (alias.obj)
+    typedef int (*init_fn)(wsrep_thread_service_v1_t*);
+    typedef void (*deinit_fn)();
+    if (wsrep_impl::service_probe<init_fn>(
+            dlh, WSREP_THREAD_SERVICE_INIT_FUNC_V1, "thread service v1") ||
+        wsrep_impl::service_probe<deinit_fn>(
+            dlh, WSREP_THREAD_SERVICE_DEINIT_FUNC_V1, "thread service v1"))
     {
-        wsrep::log_info()
-            << "Found support for thread service v1 from provider";
-        return 0;
+        wsrep::log_warning() << "Provider does not support thread service v1";
+        return 1;
     }
-    else
-    {
-        wsrep::log_info() << "Thread service v1 not found from provider: "
-                          << dlerror();
-        return ENOTSUP;
-    }
-
+    return 0;
 }
 
 int wsrep::thread_service_v1_init(void* dlh,
                                   wsrep::thread_service* thread_service)
 {
     if (not (dlh && thread_service)) return EINVAL;
-
     typedef int (*init_fn)(wsrep_thread_service_v1_t*);
-    union {
-        init_fn dlfun;
-        void* obj;
-    } alias;
-    alias.obj = dlsym(dlh, WSREP_THREAD_SERVICE_INIT_FUNC);
-    if (alias.obj)
+    wsrep_thread_service_v1::thread_service_impl = thread_service;
+    int ret(0);
+    if ((ret = wsrep_impl::service_init<init_fn>(
+             dlh, WSREP_THREAD_SERVICE_INIT_FUNC_V1,
+             &wsrep_thread_service_v1::thread_service_callbacks,
+             "thread service v1")))
     {
-        wsrep::log_info() << "Initializing process instrumentation";
-        wsrep_thread_service_v1::thread_service_impl = thread_service;
-        return (*alias.dlfun)(&wsrep_thread_service_v1::thread_service_callbacks);
+        wsrep_thread_service_v1::thread_service_impl = 0;
     }
     else
     {
-        wsrep::log_info()
-            << "Provider does not support process instrumentation";
-        return ENOTSUP;
+        ++wsrep_thread_service_v1::use_count;
+    }
+    return ret;
+}
+
+void wsrep::thread_service_v1_deinit(void* dlh)
+{
+    typedef int (*deinit_fn)();
+    wsrep_impl::service_deinit<deinit_fn>(
+        dlh, WSREP_THREAD_SERVICE_DEINIT_FUNC_V1, "thread service v1");
+    --wsrep_thread_service_v1::use_count;
+    if (wsrep_thread_service_v1::use_count == 0)
+    {
+        wsrep_thread_service_v1::thread_service_impl = 0;
     }
 }

src/thread_service_v1.hpp

@@ -42,6 +42,14 @@ namespace wsrep
      */
     int thread_service_v1_init(void* dlh,
                                wsrep::thread_service* thread_service);
+
+    /**
+     * Deinitialize the thread service.
+     *
+     * @params dlh Handler returned by dlopen().
+     */
+    void thread_service_v1_deinit(void* dlh);
+
 }
 
 #endif // WSREP_THREAD_SERVICE_V1_HPP

src/tls_service_v1.cpp

@@ -0,0 +1,232 @@
+/*
+ * Copyright (C) 2020 Codership Oy <info@codership.com>
+ *
+ * This file is part of wsrep-lib.
+ *
+ * Wsrep-lib is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * Wsrep-lib is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with wsrep-lib.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "tls_service_v1.hpp"
+
+#include "wsrep/tls_service.hpp"
+#include "wsrep/logger.hpp"
+#include "v26/wsrep_tls_service.h"
+#include "service_helpers.hpp"
+
+#include <cassert>
+
+namespace wsrep_tls_service_v1
+{
+    static wsrep::tls_service* tls_service_impl{0};
+    static std::atomic<size_t> use_count;
+
+    static int tls_stream_init_cb(
+        wsrep_tls_context_t*,
+        wsrep_tls_stream_t* stream)
+    {
+        assert(tls_service_impl);
+        stream->opaque =
+            tls_service_impl->create_tls_stream(stream->fd);
+        if (not stream->opaque)
+        {
+            return ENOMEM;
+        }
+        return 0;
+    }
+
+    static void tls_stream_deinit_cb(
+        wsrep_tls_context_t*,
+        wsrep_tls_stream_t* stream)
+    {
+        assert(tls_service_impl);
+        tls_service_impl->destroy(
+            reinterpret_cast<wsrep::tls_stream*>(stream->opaque));
+    }
+
+    static int tls_stream_get_error_number_cb(
+        wsrep_tls_context_t*,
+        const wsrep_tls_stream_t* stream)
+    {
+        assert(tls_service_impl);
+        return tls_service_impl->get_error_number(
+            reinterpret_cast<const wsrep::tls_stream*>(stream->opaque));
+    }
+
+    static const void* tls_stream_get_error_category_cb(
+        wsrep_tls_context_t*,
+        const wsrep_tls_stream_t* stream)
+    {
+        assert(tls_service_impl);
+        return tls_service_impl->get_error_category(
+            reinterpret_cast<const wsrep::tls_stream*>(stream->opaque));
+    }
+
+    static const char* tls_error_message_get_cb(
+        wsrep_tls_context_t*,
+        const wsrep_tls_stream_t* stream,
+        int value, const void* category)
+    {
+        assert(tls_service_impl);
+        return tls_service_impl->get_error_message(
+            reinterpret_cast<const wsrep::tls_stream*>(stream->opaque), value, category);
+    }
+
+    static enum wsrep_tls_result map_return_value(ssize_t status)
+    {
+        switch (status)
+        {
+        case wsrep::tls_service::success:
+            return wsrep_tls_result_success;
+        case wsrep::tls_service::want_read:
+            return wsrep_tls_result_want_read;
+        case wsrep::tls_service::want_write:
+            return wsrep_tls_result_want_write;
+        case wsrep::tls_service::eof:
+            return wsrep_tls_result_eof;
+        case wsrep::tls_service::error:
+            return wsrep_tls_result_error;
+        default:
+            assert(status < 0);
+            return wsrep_tls_result_error;
+        }
+    }
+
+
+    static enum wsrep_tls_result
+    tls_stream_client_handshake_cb(
+        wsrep_tls_context_t*,
+        wsrep_tls_stream_t* stream)
+    {
+        assert(tls_service_impl);
+        return map_return_value(
+            tls_service_impl->client_handshake(
+                reinterpret_cast<wsrep::tls_stream*>(stream->opaque)));
+    }
+
+    static enum wsrep_tls_result
+    tls_stream_server_handshake_cb(
+        wsrep_tls_context_t*,
+        wsrep_tls_stream_t* stream)
+    {
+        assert(tls_service_impl);
+        return map_return_value(
+            tls_service_impl->server_handshake(
+                reinterpret_cast<wsrep::tls_stream*>(stream->opaque)));
+    }
+
+    static enum wsrep_tls_result tls_stream_read_cb(
+        wsrep_tls_context_t*,
+        wsrep_tls_stream_t* stream,
+        void* buf,
+        size_t max_count,
+        size_t* bytes_transferred)
+    {
+        assert(tls_service_impl);
+        auto result(tls_service_impl->read(
+                        reinterpret_cast<wsrep::tls_stream*>(stream->opaque),
+                        buf, max_count));
+        *bytes_transferred = result.bytes_transferred;
+        return map_return_value(result.status);
+    }
+
+    static enum wsrep_tls_result tls_stream_write_cb(
+        wsrep_tls_context_t*,
+        wsrep_tls_stream_t* stream,
+        const void* buf,
+        size_t count,
+        size_t* bytes_transferred)
+    {
+        assert(tls_service_impl);
+        auto result(tls_service_impl->write(
+                        reinterpret_cast<wsrep::tls_stream*>(stream->opaque),
+                        buf, count));
+        *bytes_transferred = result.bytes_transferred;
+        return map_return_value(result.status);
+    }
+
+    static enum wsrep_tls_result
+    tls_stream_shutdown_cb(wsrep_tls_context_t*,
+                           wsrep_tls_stream_t* stream)
+    {
+        assert(tls_service_impl);
+        // @todo Handle other values than success.
+        return map_return_value(
+            tls_service_impl->shutdown(
+                reinterpret_cast<wsrep::tls_stream*>(stream->opaque)));
+    }
+
+    static wsrep_tls_service_v1_t tls_service_callbacks =
+    {
+        tls_stream_init_cb,
+        tls_stream_deinit_cb,
+        tls_stream_get_error_number_cb,
+        tls_stream_get_error_category_cb,
+        tls_stream_client_handshake_cb,
+        tls_stream_server_handshake_cb,
+        tls_stream_read_cb,
+        tls_stream_write_cb,
+        tls_stream_shutdown_cb,
+        tls_error_message_get_cb,
+        0 // we pass NULL context for now.
+    };
+}
+
+int wsrep::tls_service_v1_probe(void* dlh)
+{
+    typedef int (*init_fn)(wsrep_tls_service_v1_t*);
+    typedef void (*deinit_fn)();
+    if (wsrep_impl::service_probe<init_fn>(
+            dlh, WSREP_TLS_SERVICE_INIT_FUNC_V1, "tls service v1") ||
+        wsrep_impl::service_probe<deinit_fn>(
+            dlh, WSREP_TLS_SERVICE_DEINIT_FUNC_V1, "tls service v1"))
+    {
+        wsrep::log_warning() << "Provider does not support tls service v1";
+        return 1;
+    }
+    return 0;
+}
+
+int wsrep::tls_service_v1_init(void* dlh,
+                               wsrep::tls_service* tls_service)
+{
+    if (not (dlh && tls_service)) return EINVAL;
+
+    typedef int (*init_fn)(wsrep_tls_service_v1_t*);
+    wsrep_tls_service_v1::tls_service_impl = tls_service;
+    int ret(0);
+    if ((ret = wsrep_impl::service_init<init_fn>(
+             dlh, WSREP_TLS_SERVICE_INIT_FUNC_V1,
+             &wsrep_tls_service_v1::tls_service_callbacks,
+             "tls service v1")))
+    {
+        wsrep_tls_service_v1::tls_service_impl = 0;
+    }
+    else
+    {
+        ++wsrep_tls_service_v1::use_count;
+    }
+    return ret;
+}
+
+void wsrep::tls_service_v1_deinit(void* dlh)
+{
+    typedef int (*deinit_fn)();
+    wsrep_impl::service_deinit<deinit_fn>(
+        dlh, WSREP_TLS_SERVICE_DEINIT_FUNC_V1, "tls service v1");
+    --wsrep_tls_service_v1::use_count;
+    if (wsrep_tls_service_v1::use_count == 0)
+    {
+        wsrep_tls_service_v1::tls_service_impl = 0;
+    }
+}

src/tls_service_v1.hpp

@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2020 Codership Oy <info@codership.com>
+ *
+ * This file is part of wsrep-lib.
+ *
+ * Wsrep-lib is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * Wsrep-lib is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with wsrep-lib.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#ifndef WSREP_TLS_SERVICE_V1_HPP
+#define WSREP_TLS_SERVICE_V1_HPP
+
+namespace wsrep
+{
+    class tls_service;
+    /**
+     * Probe thread_service_v1 support in loaded library.
+     *
+     * @param dlh Handle returned by dlopen().
+     *
+     * @return Zero on success, non-zero system error code on failure.
+     */
+    int tls_service_v1_probe(void *dlh);
+
+    /**
+     * Initialize TLS service.
+     *
+     * @param dlh Handle returned by dlopen().
+     * @params thread_service Pointer to wsrep::thread_service implementation.
+     *
+     * @return Zero on success, non-zero system error code on failure.
+     */
+    int tls_service_v1_init(void* dlh,
+                            wsrep::tls_service* thread_service);
+
+    /**
+     * Deinitialize TLS service.
+     *
+     * @param dlh Handler returned by dlopen().
+     */
+    void tls_service_v1_deinit(void* dlh);
+}
+
+#endif // WSREP_TLS_SERVICE_V1_HPP

src/wsrep_provider_v26.cpp

@@ -26,8 +26,10 @@
 #include "wsrep/exception.hpp"
 #include "wsrep/logger.hpp"
 #include "wsrep/thread_service.hpp"
+#include "wsrep/tls_service.hpp"
 
 #include "thread_service_v1.hpp"
+#include "tls_service_v1.hpp"
 #include "v26/wsrep_api.h"
 
 
@@ -592,7 +594,7 @@ namespace
         if (wsrep::thread_service_v1_probe(dlh))
         {
             // No support in library.
-            return 0;
+            return 1;
         }
         else
         {
@@ -610,6 +612,58 @@ namespace
         }
         return 0;
     }
+
+    static void deinit_thread_service(void* dlh)
+    {
+        // assert(not wsrep::thread_service_v1_probe(dlh));
+        wsrep::thread_service_v1_deinit(dlh);
+    }
+
+    static int init_tls_service(void* dlh,
+                                wsrep::tls_service* tls_service)
+    {
+        assert(tls_service);
+        if (not wsrep::tls_service_v1_probe(dlh))
+        {
+            return wsrep::tls_service_v1_init(dlh, tls_service);
+        }
+        return 1;
+    }
+
+    static void deinit_tls_service(void* dlh)
+    {
+        // assert(not wsrep::tls_service_v1_probe(dlh));
+        wsrep::tls_service_v1_deinit(dlh);
+    }
+}
+
+void wsrep::wsrep_provider_v26::init_services(
+    const wsrep::provider::services& services)
+{
+    if (services.thread_service)
+    {
+        if (init_thread_service(wsrep_->dlh, services.thread_service))
+        {
+            throw wsrep::runtime_error("Failed to initialize thread service");
+        }
+        services_enabled_.thread_service = services.thread_service;
+    }
+    if (services.tls_service)
+    {
+        if (init_tls_service(wsrep_->dlh, services.tls_service))
+        {
+            throw wsrep::runtime_error("Failed to initialze TLS service");
+        }
+        services_enabled_.tls_service = services.tls_service;
+    }
+}
+
+void wsrep::wsrep_provider_v26::deinit_services()
+{
+    if (services_enabled_.tls_service)
+        deinit_tls_service(wsrep_->dlh);
+    if (services_enabled_.thread_service)
+        deinit_thread_service(wsrep_->dlh);
 }
 
 wsrep::wsrep_provider_v26::wsrep_provider_v26(
@@ -619,6 +673,7 @@ wsrep::wsrep_provider_v26::wsrep_provider_v26(
     const wsrep::provider::services& services)
     : provider(server_state)
     , wsrep_()
+    , services_enabled_()
 {
     wsrep_gtid_t state_id;
     bool encryption_enabled = server_state.encryption_service() &&
@@ -653,11 +708,7 @@ wsrep::wsrep_provider_v26::wsrep_provider_v26(
         throw wsrep::runtime_error("Failed to load wsrep library");
     }
 
-    if (services.thread_service &&
-        init_thread_service(wsrep_->dlh, services.thread_service))
-    {
-        throw wsrep::runtime_error("Failed to initialize thread service");
-    }
+    init_services(services);
 
     if (wsrep_->init(wsrep_, &init_args) != WSREP_OK)
     {
@@ -683,6 +734,8 @@ wsrep::wsrep_provider_v26::wsrep_provider_v26(
 
 wsrep::wsrep_provider_v26::~wsrep_provider_v26()
 {
+    wsrep_->free(wsrep_);
+    deinit_services();
     wsrep_unload(wsrep_);
 }
 

src/wsrep_provider_v26.hpp

@@ -30,7 +30,8 @@ namespace wsrep
     class wsrep_provider_v26 : public wsrep::provider
     {
     public:
-
+        void init_services(const wsrep::provider::services& services);
+        void deinit_services();
         wsrep_provider_v26(wsrep::server_state&, const std::string&,
                            const std::string&,
                            const wsrep::provider::services& services);
@@ -107,6 +108,7 @@ namespace wsrep
         wsrep_provider_v26(const wsrep_provider_v26&);
         wsrep_provider_v26& operator=(const wsrep_provider_v26);
         struct wsrep_st* wsrep_;
+        services services_enabled_;
     };
 }