database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-04-18 21:44:02 +03:00
Code

MCOL-5719: Move ownership mechanism to KV storage. (#3266)

Browse Source 
* MCOL-5719 Move ownership mechanism to FDB
This commit is contained in:
Denis Khalikov 2024-11-09 22:47:04 +03:00 committed by GitHub
parent 8ae5a3da40
commit a6eb5ca689
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 287 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CMakeLists.txt
View File

@ -357,6 +357,7 @@ SET (ENGINE_UTILS_BATCHLDR_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/batchlo
SET (ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlcleanup")
SET (ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats")
SET (ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client")
SET (ENGINE_UTILS_FDB_WRAPPER "${CMAKE_CURRENT_SOURCE_DIR}/utils/fdb_wrapper_cpp/include")
SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml")
SET (ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes")
SET (ENGINE_BLOCKCACHE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/blockcache")
@ -374,7 +375,7 @@ SET (ENGINE_UTILS_UDFSDK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/udfsdk"
SET (ENGINE_DEFAULT_INCLUDES ${CMAKE_CURRENT_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR})
SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE})
SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE} ${ENGINE_UTILS_FDB_WRAPPER})
ADD_SUBDIRECTORY(dbcon/mysql)
IF(NOT TARGET columnstore)

1
build/bootstrap_mcs.sh
View File

@ -212,6 +212,7 @@ clean_old_installation()
rm -rf /etc/mysql
rm -rf /etc/my.cnf.d/columnstore.cnf
rm -rf /etc/mysql/mariadb.conf.d/columnstore.cnf
fdbcli --exec "writemode on; clearrange SM_A SM_z"
}
build()

8
storage-manager/CMakeLists.txt
View File

@ -38,6 +38,8 @@ set(storagemanager_SRCS
src/SyncTask.cpp
src/ListIOTask.cpp
src/TerminateIOTask.cpp
src/KVStorageInitializer.cpp
src/KVPrefixes.cpp
../utils/common/crashtrace.cpp
)
@ -62,13 +64,11 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/storagemanager.cnf.in" "${CMAKE_CURR
link_directories(${CMAKE_BINARY_DIR}/lib)
set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib)
add_library(storagemanager SHARED ${storagemanager_SRCS})
add_dependencies(storagemanager marias3 external_boost)
add_dependencies(storagemanager marias3 external_boost fdbcs)
target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
target_link_libraries(storagemanager boost_chrono boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS})
target_link_libraries(storagemanager boost_chrono boost_system boost_thread boost_filesystem boost_regex pthread fdbcs ${S3API_DEPS})
add_executable(StorageManager src/main.cpp)
target_link_libraries(StorageManager storagemanager)

24
storage-manager/src/KVPrefixes.cpp Normal file
View File

@ -0,0 +1,24 @@
/* Copyright (C) 2024 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "KVPrefixes.hpp"
namespace storagemanager
{
// FDB recommends keep the key size up to 32 bytes.
const char* KVPrefixes[2] = {/*OWNERSHIP*/ "SM_O_", /*META*/ "SM_M_"};
} // namespace storagemanager

29
storage-manager/src/KVPrefixes.hpp Normal file
View File

@ -0,0 +1,29 @@
/* Copyright (C) 2024 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
namespace storagemanager
{
enum class KVPrefixId
{
SM_OWNERSHIP = 0,
SM_META
};
extern const char* KVPrefixes[];
} // namespace storagemanager

64
storage-manager/src/KVStorageInitializer.cpp Normal file
View File

@ -0,0 +1,64 @@
/* Copyright (C) 2024 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "Config.h"
#include "SMLogging.h"
#include "KVStorageInitializer.h"
namespace storagemanager
{
static std::shared_ptr<FDBCS::FDBDataBase> fdbDataBaseInstance;
static std::unique_ptr<FDBCS::FDBNetwork> fdbNetworkInstance;
static std::mutex kvStorageLock;
static void logError(SMLogging* logger, const char* errorMsg)
{
logger->log(LOG_CRIT, errorMsg);
throw std::runtime_error(errorMsg);
}
std::shared_ptr<FDBCS::FDBDataBase> KVStorageInitializer::getStorageInstance()
{
if (fdbDataBaseInstance)
return fdbDataBaseInstance;
auto* config = Config::get();
auto* logger = SMLogging::get();
std::unique_lock<std::mutex> lock(kvStorageLock);
if (fdbDataBaseInstance)
return fdbDataBaseInstance;
if (!FDBCS::setAPIVersion())
logError(logger, "Ownership: FDB setAPIVersion failed.");
fdbNetworkInstance = std::make_unique<FDBCS::FDBNetwork>();
if (!fdbNetworkInstance->setUpAndRunNetwork())
logError(logger, "Ownership: FDB setUpAndRunNetwork failed.");
std::string clusterFilePath = config->getValue("ObjectStorage", "fdb_cluster_file_path");
if (clusterFilePath.empty())
logError(logger,
"Ownership: Need to specify `Foundation DB cluster file path` in the storagemanager.cnf file");
fdbDataBaseInstance = FDBCS::DataBaseCreator::createDataBase(clusterFilePath);
if (!fdbDataBaseInstance)
logError(logger, "Ownership: FDB createDataBase failed.");
return fdbDataBaseInstance;
}
} // namespace storagemanager

38
storage-manager/src/KVStorageInitializer.h Normal file
View File

@ -0,0 +1,38 @@
/* Copyright (C) 2024 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <memory>
#include "fdbcs.hpp"
namespace storagemanager
{
// Represnts a `Key/Value` storage initializer based on Singleton pattern.
// Initializes all needed `Key/Value` storage machinery once and return pointer
// to the `Key/Value` storage.
class KVStorageInitializer
{
public:
static std::shared_ptr<FDBCS::FDBDataBase> getStorageInstance();
KVStorageInitializer() = delete;
KVStorageInitializer(const KVStorageInitializer&) = delete;
KVStorageInitializer& operator=(KVStorageInitializer&) = delete;
KVStorageInitializer(KVStorageInitializer&&) = delete;
KVStorageInitializer& operator=(KVStorageInitializer&&) = delete;
};
} // namespace storagemanager

177
storage-manager/src/Ownership.cpp
View File

@ -1,4 +1,4 @@
/* Copyright (C) 2019 MariaDB Corporation
/* Copyright (C) 2024 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
@ -19,6 +19,9 @@
#include "Config.h"
#include "Cache.h"
#include "Synchronizer.h"
#include "KVStorageInitializer.h"
#include "KVPrefixes.hpp"
#include "fdbcs.hpp"
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
@ -30,6 +33,31 @@ namespace bf = boost::filesystem;
namespace storagemanager
{
// FDB recommends keep the key size up to 32 bytes.
const char* ownerShipStates[3] = {/*OWNED*/ "_O", /*FLUSHING*/ "_F", /*REQUEST_TRANSFER*/ "_RT"};
inline std::string getKeyName(const std::string& fileName, OwnershipStateId state)
{
return KVPrefixes[static_cast<uint32_t>(KVPrefixId::SM_OWNERSHIP)] + fileName +
ownerShipStates[static_cast<uint32_t>(state)];
}
inline void Ownership::removeKeys(const std::string& fileName, const std::vector<OwnershipStateId>& states)
{
auto kvStorage = KVStorageInitializer::getStorageInstance();
auto tnx = kvStorage->createTransaction();
for (const auto state : states)
tnx->remove(getKeyName(fileName, state));
if (!tnx->commit())
{
const char* msg = "Ownership: commit `removeKeys` failed ";
logger->log(LOG_CRIT, msg);
throw runtime_error(msg);
}
}
Ownership::Ownership()
{
Config* config = Config::get();
@ -61,6 +89,7 @@ Ownership::Ownership()
logger->log(LOG_CRIT, msg);
throw runtime_error(msg);
}
monitor = new Monitor(this);
}
@ -124,28 +153,22 @@ bf::path Ownership::get(const bf::path& p, bool getOwnership)
return ret;
}
// minor timesaver
#define TOUCH(p, f) \
{ \
int fd = ::open((metadataPrefix / p / f).string().c_str(), O_TRUNC | O_CREAT | O_WRONLY, 0660); \
if (fd >= 0) \
::close(fd); \
else \
{ \
char buf[80]; \
int saved_errno = errno; \
cerr << "failed to touch " << metadataPrefix / p / f << " got " << strerror_r(saved_errno, buf, 80) \
<< endl; \
} \
}
#define DELETE(p, f) ::unlink((metadataPrefix / p / f).string().c_str());
void Ownership::touchFlushing(const bf::path& prefix, volatile bool* doneFlushing) const
{
while (!*doneFlushing)
{
TOUCH(prefix, "FLUSHING");
{
auto kvStorage = KVStorageInitializer::getStorageInstance();
auto tnx = kvStorage->createTransaction();
const std::string flushingKey = getKeyName(prefix.string(), OwnershipStateId::FLUSHING);
tnx->set(flushingKey, "");
if (!tnx->commit())
{
const char* msg = "Ownership: commit `touchFlushing` failed ";
logger->log(LOG_CRIT, msg);
throw runtime_error(msg);
}
}
try
{
boost::this_thread::sleep_for(boost::chrono::seconds(1));
@ -171,10 +194,7 @@ void Ownership::releaseOwnership(const bf::path& p, bool isDtor)
if (isDtor)
{
// This is a quick release. If this is being destroyed, then it is through the graceful
// shutdown mechanism, which will flush data separately.
DELETE(p, "OWNED");
DELETE(p, "FLUSHING");
removeKeys(p.string(), {OwnershipStateId::OWNED, OwnershipStateId::FLUSHING});
return;
}
else
@ -191,19 +211,28 @@ void Ownership::releaseOwnership(const bf::path& p, bool isDtor)
done = true;
xfer.interrupt();
xfer.join();
// update state
DELETE(p, "OWNED");
DELETE(p, "FLUSHING");
removeKeys(p.string(), {OwnershipStateId::OWNED, OwnershipStateId::FLUSHING});
}
void Ownership::_takeOwnership(const bf::path& p)
{
logger->log(LOG_DEBUG, "Ownership: taking ownership of %s", p.string().c_str());
DELETE(p, "FLUSHING");
DELETE(p, "REQUEST_TRANSFER");
// TODO: need to consider errors taking ownership
TOUCH(p, "OWNED");
{
auto kvStorage = KVStorageInitializer::getStorageInstance();
auto tnx = kvStorage->createTransaction();
const std::string ownedKey = getKeyName(p.string(), OwnershipStateId::OWNED);
const std::string flushingKey = getKeyName(p.string(), OwnershipStateId::FLUSHING);
const std::string requestTransferKey = getKeyName(p.string(), OwnershipStateId::REQUEST_TRANSFER);
tnx->remove(flushingKey);
tnx->remove(requestTransferKey);
tnx->set(ownedKey, "");
if (!tnx->commit())
{
const char* msg = "Ownership: commit `_takeOwnership` transaction failed";
logger->log(LOG_CRIT, msg);
throw runtime_error(msg);
}
}
mutex.lock();
ownedPrefixes[p] = true;
mutex.unlock();
@ -225,45 +254,62 @@ void Ownership::takeOwnership(const bf::path& p)
ownedPrefixes[p] = NULL;
s.unlock();
bool okToTransfer = false;
struct stat statbuf;
int err;
char buf[80];
bf::path ownedPath = metadataPrefix / p / "OWNED";
bf::path flushingPath = metadataPrefix / p / "FLUSHING";
bool ownedKeyExists;
{
auto kvStorage = KVStorageInitializer::getStorageInstance();
auto tnx = kvStorage->createTransaction();
const std::string ownedKey = getKeyName(p.string(), OwnershipStateId::OWNED);
ownedKeyExists = tnx->get(ownedKey).first;
}
// if it's not already owned, then we can take possession
err = ::stat(ownedPath.string().c_str(), &statbuf);
if (err && errno == ENOENT)
if (!ownedKeyExists)
{
_takeOwnership(p);
return;
}
TOUCH(p, "REQUEST_TRANSFER");
{
auto kvStorage = KVStorageInitializer::getStorageInstance();
auto tnx = kvStorage->createTransaction();
const std::string requestTransferKey = getKeyName(p.string(), OwnershipStateId::REQUEST_TRANSFER);
tnx->set(requestTransferKey, "");
if (!tnx->commit())
{
const char* msg = "Ownership: commit `requestTransfer` failed ";
logger->log(LOG_CRIT, msg);
throw runtime_error(msg);
}
}
bool okToTransfer = false;
time_t lastFlushTime = time(NULL);
while (!okToTransfer && time(NULL) < lastFlushTime + 10)
{
// if the OWNED file is deleted or if the flushing file isn't touched after 10 secs
// it is ok to take possession.
err = ::stat(ownedPath.string().c_str(), &statbuf);
if (err)
bool ownedKeyExists;
{
if (errno == ENOENT)
okToTransfer = true;
else
logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s",
strerror_r(errno, buf, 80), ownedPath.string().c_str());
auto kvStorage = KVStorageInitializer::getStorageInstance();
auto tnx = kvStorage->createTransaction();
const std::string ownedKey = getKeyName(p.string(), OwnershipStateId::OWNED);
ownedKeyExists = tnx->get(ownedKey).first;
}
err = ::stat(flushingPath.string().c_str(), &statbuf);
if (err && errno != ENOENT)
logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s",
strerror_r(errno, buf, 80), flushingPath.string().c_str());
else
if (!ownedKeyExists)
okToTransfer = true;
bool flushingKeyExists;
{
auto kvStorage = KVStorageInitializer::getStorageInstance();
auto tnx = kvStorage->createTransaction();
const std::string flushingKey = getKeyName(p.string(), OwnershipStateId::FLUSHING);
flushingKeyExists = tnx->get(flushingKey).first;
}
if (flushingKeyExists)
{
logger->log(LOG_DEBUG, "Ownership: waiting to get %s", p.string().c_str());
if (!err)
lastFlushTime = statbuf.st_mtime;
// Since notice the flushing key.
lastFlushTime = time(NULL);
}
if (!okToTransfer)
sleep(1);
@ -286,9 +332,6 @@ Ownership::Monitor::~Monitor()
void Ownership::Monitor::watchForInterlopers()
{
// look for requests to transfer ownership
struct stat statbuf;
int err;
char buf[80];
vector<bf::path> releaseList;
while (!stop)
@ -302,17 +345,20 @@ void Ownership::Monitor::watchForInterlopers()
break;
if (prefix.second == false)
continue;
bf::path p(owner->metadataPrefix / (prefix.first) / "REQUEST_TRANSFER");
const char* cp = p.string().c_str();
err = ::stat(cp, &statbuf);
bool requestKeyExists;
{
auto kvStorage = KVStorageInitializer::getStorageInstance();
auto tnx = kvStorage->createTransaction();
const std::string requestTransferKey =
getKeyName(prefix.first.string(), OwnershipStateId::REQUEST_TRANSFER);
requestKeyExists = tnx->get(requestTransferKey).first;
}
// release it if there's a release request only. Log it if there's an error other than
// that the file isn't there.
if (err == 0)
if (requestKeyExists)
releaseList.push_back(prefix.first);
if (err < 0 && errno != ENOENT)
owner->logger->log(LOG_ERR, "Runner::watchForInterlopers(): failed to stat %s, got %s", cp,
strerror_r(errno, buf, 80));
}
s.unlock();
@ -329,5 +375,4 @@ void Ownership::Monitor::watchForInterlopers()
}
}
}
} // namespace storagemanager
} // namespace storagemanager

9
storage-manager/src/Ownership.h
View File

@ -22,12 +22,20 @@
#include <boost/thread.hpp>
#include <map>
#include "SMLogging.h"
#include "fdbcs.hpp"
/* This class tracks the ownership of each prefix and manages ownership transfer.
Could we come up with a better name btw? */
namespace storagemanager
{
enum class OwnershipStateId
{
OWNED = 0,
FLUSHING,
REQUEST_TRANSFER
};
class Ownership : public boost::noncopyable
{
public:
@ -50,6 +58,7 @@ class Ownership : public boost::noncopyable
void takeOwnership(const boost::filesystem::path&);
void releaseOwnership(const boost::filesystem::path&, bool isDtor = false);
void _takeOwnership(const boost::filesystem::path&);
inline void removeKeys(const std::string& fileName, const std::vector<OwnershipStateId>& states);
struct Monitor
{

4
storage-manager/storagemanager.cnf.in
View File

@ -80,6 +80,10 @@ max_concurrent_uploads = 21
# negotiate ownership of data from a failed instance.
common_prefix_depth = 3
# This value is used to specify a cluster file path to FDB cluster.
# See more: https://apple.github.io/foundationdb/administration.html
fdb_cluster_file_path = /etc/foundationdb/fdb.cluster
[S3]
# These should be self-explanatory. Region can be blank or commented
# if using a private cloud storage system. Bucket has to be set to

2
utils/fdb_wrapper_cpp/test/test_fdb_api.cpp
View File

@ -16,7 +16,7 @@
MA 02110-1301, USA. */
#include <vector>
#include "../include/fdbcs.hpp"
#include "fdbcs.hpp"
using namespace std;
using namespace FDBCS;