diff --git a/CMakeLists.txt b/CMakeLists.txt index 70f9c0b8b..0f8cd0737 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -357,6 +357,7 @@ SET (ENGINE_UTILS_BATCHLDR_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/batchlo SET (ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlcleanup") SET (ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats") SET (ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client") +SET (ENGINE_UTILS_FDB_WRAPPER "${CMAKE_CURRENT_SOURCE_DIR}/utils/fdb_wrapper_cpp/include") SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml") SET (ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes") SET (ENGINE_BLOCKCACHE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/blockcache") @@ -374,7 +375,7 @@ SET (ENGINE_UTILS_UDFSDK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/udfsdk" SET (ENGINE_DEFAULT_INCLUDES ${CMAKE_CURRENT_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR}) -SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE}) +SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE} ${ENGINE_UTILS_FDB_WRAPPER}) ADD_SUBDIRECTORY(dbcon/mysql) IF(NOT TARGET columnstore) diff --git a/build/bootstrap_mcs.sh b/build/bootstrap_mcs.sh index 3bae32e9d..a95901b3b 100755 --- a/build/bootstrap_mcs.sh +++ b/build/bootstrap_mcs.sh @@ -212,6 +212,7 @@ clean_old_installation() rm -rf /etc/mysql rm -rf /etc/my.cnf.d/columnstore.cnf rm -rf /etc/mysql/mariadb.conf.d/columnstore.cnf + fdbcli --exec "writemode on; clearrange SM_A SM_z" } build() diff --git a/storage-manager/CMakeLists.txt b/storage-manager/CMakeLists.txt index 4297081c5..707595b70 100755 --- a/storage-manager/CMakeLists.txt +++ b/storage-manager/CMakeLists.txt @@ -38,6 +38,8 @@ set(storagemanager_SRCS src/SyncTask.cpp src/ListIOTask.cpp src/TerminateIOTask.cpp + src/KVStorageInitializer.cpp + src/KVPrefixes.cpp ../utils/common/crashtrace.cpp ) @@ -62,13 +64,11 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/storagemanager.cnf.in" "${CMAKE_CURR link_directories(${CMAKE_BINARY_DIR}/lib) set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib) - - add_library(storagemanager SHARED ${storagemanager_SRCS}) -add_dependencies(storagemanager marias3 external_boost) +add_dependencies(storagemanager marias3 external_boost fdbcs) target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS) -target_link_libraries(storagemanager boost_chrono boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS}) +target_link_libraries(storagemanager boost_chrono boost_system boost_thread boost_filesystem boost_regex pthread fdbcs ${S3API_DEPS}) add_executable(StorageManager src/main.cpp) target_link_libraries(StorageManager storagemanager) diff --git a/storage-manager/src/KVPrefixes.cpp b/storage-manager/src/KVPrefixes.cpp new file mode 100644 index 000000000..8183d9f69 --- /dev/null +++ b/storage-manager/src/KVPrefixes.cpp @@ -0,0 +1,24 @@ +/* Copyright (C) 2024 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#include "KVPrefixes.hpp" + +namespace storagemanager +{ +// FDB recommends keep the key size up to 32 bytes. +const char* KVPrefixes[2] = {/*OWNERSHIP*/ "SM_O_", /*META*/ "SM_M_"}; +} // namespace storagemanager diff --git a/storage-manager/src/KVPrefixes.hpp b/storage-manager/src/KVPrefixes.hpp new file mode 100644 index 000000000..92d8b191c --- /dev/null +++ b/storage-manager/src/KVPrefixes.hpp @@ -0,0 +1,29 @@ +/* Copyright (C) 2024 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once + +namespace storagemanager +{ +enum class KVPrefixId +{ + SM_OWNERSHIP = 0, + SM_META +}; + +extern const char* KVPrefixes[]; +} // namespace storagemanager diff --git a/storage-manager/src/KVStorageInitializer.cpp b/storage-manager/src/KVStorageInitializer.cpp new file mode 100644 index 000000000..77f9af8a3 --- /dev/null +++ b/storage-manager/src/KVStorageInitializer.cpp @@ -0,0 +1,64 @@ +/* Copyright (C) 2024 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ +#include "Config.h" +#include "SMLogging.h" +#include "KVStorageInitializer.h" + +namespace storagemanager +{ +static std::shared_ptr fdbDataBaseInstance; +static std::unique_ptr fdbNetworkInstance; +static std::mutex kvStorageLock; + +static void logError(SMLogging* logger, const char* errorMsg) +{ + logger->log(LOG_CRIT, errorMsg); + throw std::runtime_error(errorMsg); +} + +std::shared_ptr KVStorageInitializer::getStorageInstance() +{ + if (fdbDataBaseInstance) + return fdbDataBaseInstance; + + auto* config = Config::get(); + auto* logger = SMLogging::get(); + + std::unique_lock lock(kvStorageLock); + if (fdbDataBaseInstance) + return fdbDataBaseInstance; + + if (!FDBCS::setAPIVersion()) + logError(logger, "Ownership: FDB setAPIVersion failed."); + + fdbNetworkInstance = std::make_unique(); + if (!fdbNetworkInstance->setUpAndRunNetwork()) + logError(logger, "Ownership: FDB setUpAndRunNetwork failed."); + + std::string clusterFilePath = config->getValue("ObjectStorage", "fdb_cluster_file_path"); + if (clusterFilePath.empty()) + logError(logger, + "Ownership: Need to specify `Foundation DB cluster file path` in the storagemanager.cnf file"); + + fdbDataBaseInstance = FDBCS::DataBaseCreator::createDataBase(clusterFilePath); + if (!fdbDataBaseInstance) + logError(logger, "Ownership: FDB createDataBase failed."); + + return fdbDataBaseInstance; +} + +} // namespace storagemanager diff --git a/storage-manager/src/KVStorageInitializer.h b/storage-manager/src/KVStorageInitializer.h new file mode 100644 index 000000000..cd1d829a0 --- /dev/null +++ b/storage-manager/src/KVStorageInitializer.h @@ -0,0 +1,38 @@ +/* Copyright (C) 2024 MariaDB Corporation + + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; version 2 of + the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, + MA 02110-1301, USA. */ + +#pragma once +#include +#include "fdbcs.hpp" + +namespace storagemanager +{ +// Represnts a `Key/Value` storage initializer based on Singleton pattern. +// Initializes all needed `Key/Value` storage machinery once and return pointer +// to the `Key/Value` storage. +class KVStorageInitializer +{ + public: + static std::shared_ptr getStorageInstance(); + KVStorageInitializer() = delete; + KVStorageInitializer(const KVStorageInitializer&) = delete; + KVStorageInitializer& operator=(KVStorageInitializer&) = delete; + KVStorageInitializer(KVStorageInitializer&&) = delete; + KVStorageInitializer& operator=(KVStorageInitializer&&) = delete; +}; + +} // namespace storagemanager diff --git a/storage-manager/src/Ownership.cpp b/storage-manager/src/Ownership.cpp index cadcebc22..0fe670b76 100644 --- a/storage-manager/src/Ownership.cpp +++ b/storage-manager/src/Ownership.cpp @@ -1,4 +1,4 @@ -/* Copyright (C) 2019 MariaDB Corporation +/* Copyright (C) 2024 MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License @@ -19,6 +19,9 @@ #include "Config.h" #include "Cache.h" #include "Synchronizer.h" +#include "KVStorageInitializer.h" +#include "KVPrefixes.hpp" +#include "fdbcs.hpp" #include #include #include @@ -30,6 +33,31 @@ namespace bf = boost::filesystem; namespace storagemanager { +// FDB recommends keep the key size up to 32 bytes. +const char* ownerShipStates[3] = {/*OWNED*/ "_O", /*FLUSHING*/ "_F", /*REQUEST_TRANSFER*/ "_RT"}; + +inline std::string getKeyName(const std::string& fileName, OwnershipStateId state) +{ + return KVPrefixes[static_cast(KVPrefixId::SM_OWNERSHIP)] + fileName + + ownerShipStates[static_cast(state)]; +} + +inline void Ownership::removeKeys(const std::string& fileName, const std::vector& states) +{ + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + + for (const auto state : states) + tnx->remove(getKeyName(fileName, state)); + + if (!tnx->commit()) + { + const char* msg = "Ownership: commit `removeKeys` failed "; + logger->log(LOG_CRIT, msg); + throw runtime_error(msg); + } +} + Ownership::Ownership() { Config* config = Config::get(); @@ -61,6 +89,7 @@ Ownership::Ownership() logger->log(LOG_CRIT, msg); throw runtime_error(msg); } + monitor = new Monitor(this); } @@ -124,28 +153,22 @@ bf::path Ownership::get(const bf::path& p, bool getOwnership) return ret; } -// minor timesaver -#define TOUCH(p, f) \ - { \ - int fd = ::open((metadataPrefix / p / f).string().c_str(), O_TRUNC | O_CREAT | O_WRONLY, 0660); \ - if (fd >= 0) \ - ::close(fd); \ - else \ - { \ - char buf[80]; \ - int saved_errno = errno; \ - cerr << "failed to touch " << metadataPrefix / p / f << " got " << strerror_r(saved_errno, buf, 80) \ - << endl; \ - } \ - } - -#define DELETE(p, f) ::unlink((metadataPrefix / p / f).string().c_str()); - void Ownership::touchFlushing(const bf::path& prefix, volatile bool* doneFlushing) const { while (!*doneFlushing) { - TOUCH(prefix, "FLUSHING"); + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string flushingKey = getKeyName(prefix.string(), OwnershipStateId::FLUSHING); + tnx->set(flushingKey, ""); + if (!tnx->commit()) + { + const char* msg = "Ownership: commit `touchFlushing` failed "; + logger->log(LOG_CRIT, msg); + throw runtime_error(msg); + } + } try { boost::this_thread::sleep_for(boost::chrono::seconds(1)); @@ -171,10 +194,7 @@ void Ownership::releaseOwnership(const bf::path& p, bool isDtor) if (isDtor) { - // This is a quick release. If this is being destroyed, then it is through the graceful - // shutdown mechanism, which will flush data separately. - DELETE(p, "OWNED"); - DELETE(p, "FLUSHING"); + removeKeys(p.string(), {OwnershipStateId::OWNED, OwnershipStateId::FLUSHING}); return; } else @@ -191,19 +211,28 @@ void Ownership::releaseOwnership(const bf::path& p, bool isDtor) done = true; xfer.interrupt(); xfer.join(); - - // update state - DELETE(p, "OWNED"); - DELETE(p, "FLUSHING"); + removeKeys(p.string(), {OwnershipStateId::OWNED, OwnershipStateId::FLUSHING}); } void Ownership::_takeOwnership(const bf::path& p) { logger->log(LOG_DEBUG, "Ownership: taking ownership of %s", p.string().c_str()); - DELETE(p, "FLUSHING"); - DELETE(p, "REQUEST_TRANSFER"); - // TODO: need to consider errors taking ownership - TOUCH(p, "OWNED"); + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string ownedKey = getKeyName(p.string(), OwnershipStateId::OWNED); + const std::string flushingKey = getKeyName(p.string(), OwnershipStateId::FLUSHING); + const std::string requestTransferKey = getKeyName(p.string(), OwnershipStateId::REQUEST_TRANSFER); + tnx->remove(flushingKey); + tnx->remove(requestTransferKey); + tnx->set(ownedKey, ""); + if (!tnx->commit()) + { + const char* msg = "Ownership: commit `_takeOwnership` transaction failed"; + logger->log(LOG_CRIT, msg); + throw runtime_error(msg); + } + } mutex.lock(); ownedPrefixes[p] = true; mutex.unlock(); @@ -225,45 +254,62 @@ void Ownership::takeOwnership(const bf::path& p) ownedPrefixes[p] = NULL; s.unlock(); - bool okToTransfer = false; - struct stat statbuf; - int err; - char buf[80]; - bf::path ownedPath = metadataPrefix / p / "OWNED"; - bf::path flushingPath = metadataPrefix / p / "FLUSHING"; + bool ownedKeyExists; + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string ownedKey = getKeyName(p.string(), OwnershipStateId::OWNED); + ownedKeyExists = tnx->get(ownedKey).first; + } // if it's not already owned, then we can take possession - err = ::stat(ownedPath.string().c_str(), &statbuf); - if (err && errno == ENOENT) + if (!ownedKeyExists) { _takeOwnership(p); return; } - TOUCH(p, "REQUEST_TRANSFER"); + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string requestTransferKey = getKeyName(p.string(), OwnershipStateId::REQUEST_TRANSFER); + tnx->set(requestTransferKey, ""); + if (!tnx->commit()) + { + const char* msg = "Ownership: commit `requestTransfer` failed "; + logger->log(LOG_CRIT, msg); + throw runtime_error(msg); + } + } + + bool okToTransfer = false; time_t lastFlushTime = time(NULL); while (!okToTransfer && time(NULL) < lastFlushTime + 10) { // if the OWNED file is deleted or if the flushing file isn't touched after 10 secs // it is ok to take possession. - err = ::stat(ownedPath.string().c_str(), &statbuf); - if (err) + bool ownedKeyExists; { - if (errno == ENOENT) - okToTransfer = true; - else - logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s", - strerror_r(errno, buf, 80), ownedPath.string().c_str()); + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string ownedKey = getKeyName(p.string(), OwnershipStateId::OWNED); + ownedKeyExists = tnx->get(ownedKey).first; } - err = ::stat(flushingPath.string().c_str(), &statbuf); - if (err && errno != ENOENT) - logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s", - strerror_r(errno, buf, 80), flushingPath.string().c_str()); - else + if (!ownedKeyExists) + okToTransfer = true; + + bool flushingKeyExists; + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string flushingKey = getKeyName(p.string(), OwnershipStateId::FLUSHING); + flushingKeyExists = tnx->get(flushingKey).first; + } + if (flushingKeyExists) { logger->log(LOG_DEBUG, "Ownership: waiting to get %s", p.string().c_str()); - if (!err) - lastFlushTime = statbuf.st_mtime; + // Since notice the flushing key. + lastFlushTime = time(NULL); } if (!okToTransfer) sleep(1); @@ -286,9 +332,6 @@ Ownership::Monitor::~Monitor() void Ownership::Monitor::watchForInterlopers() { // look for requests to transfer ownership - struct stat statbuf; - int err; - char buf[80]; vector releaseList; while (!stop) @@ -302,17 +345,20 @@ void Ownership::Monitor::watchForInterlopers() break; if (prefix.second == false) continue; - bf::path p(owner->metadataPrefix / (prefix.first) / "REQUEST_TRANSFER"); - const char* cp = p.string().c_str(); - err = ::stat(cp, &statbuf); + bool requestKeyExists; + { + auto kvStorage = KVStorageInitializer::getStorageInstance(); + auto tnx = kvStorage->createTransaction(); + const std::string requestTransferKey = + getKeyName(prefix.first.string(), OwnershipStateId::REQUEST_TRANSFER); + + requestKeyExists = tnx->get(requestTransferKey).first; + } // release it if there's a release request only. Log it if there's an error other than // that the file isn't there. - if (err == 0) + if (requestKeyExists) releaseList.push_back(prefix.first); - if (err < 0 && errno != ENOENT) - owner->logger->log(LOG_ERR, "Runner::watchForInterlopers(): failed to stat %s, got %s", cp, - strerror_r(errno, buf, 80)); } s.unlock(); @@ -329,5 +375,4 @@ void Ownership::Monitor::watchForInterlopers() } } } - -} // namespace storagemanager +} // namespace storagemanager \ No newline at end of file diff --git a/storage-manager/src/Ownership.h b/storage-manager/src/Ownership.h index f1e3a5c74..f3b6a2d97 100644 --- a/storage-manager/src/Ownership.h +++ b/storage-manager/src/Ownership.h @@ -22,12 +22,20 @@ #include #include #include "SMLogging.h" +#include "fdbcs.hpp" /* This class tracks the ownership of each prefix and manages ownership transfer. Could we come up with a better name btw? */ namespace storagemanager { +enum class OwnershipStateId +{ + OWNED = 0, + FLUSHING, + REQUEST_TRANSFER +}; + class Ownership : public boost::noncopyable { public: @@ -50,6 +58,7 @@ class Ownership : public boost::noncopyable void takeOwnership(const boost::filesystem::path&); void releaseOwnership(const boost::filesystem::path&, bool isDtor = false); void _takeOwnership(const boost::filesystem::path&); + inline void removeKeys(const std::string& fileName, const std::vector& states); struct Monitor { diff --git a/storage-manager/storagemanager.cnf.in b/storage-manager/storagemanager.cnf.in index 04347aced..8999c7e63 100644 --- a/storage-manager/storagemanager.cnf.in +++ b/storage-manager/storagemanager.cnf.in @@ -80,6 +80,10 @@ max_concurrent_uploads = 21 # negotiate ownership of data from a failed instance. common_prefix_depth = 3 +# This value is used to specify a cluster file path to FDB cluster. +# See more: https://apple.github.io/foundationdb/administration.html +fdb_cluster_file_path = /etc/foundationdb/fdb.cluster + [S3] # These should be self-explanatory. Region can be blank or commented # if using a private cloud storage system. Bucket has to be set to diff --git a/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp b/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp index 98ca763f4..dc9edb3e7 100644 --- a/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp +++ b/utils/fdb_wrapper_cpp/test/test_fdb_api.cpp @@ -16,7 +16,7 @@ MA 02110-1301, USA. */ #include -#include "../include/fdbcs.hpp" +#include "fdbcs.hpp" using namespace std; using namespace FDBCS;