You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-07-07 09:01:10 +03:00
MCOL-5719: Move ownership mechanism to KV storage. (#3266)
* MCOL-5719 Move ownership mechanism to FDB
This commit is contained in:
@ -357,6 +357,7 @@ SET (ENGINE_UTILS_BATCHLDR_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/batchlo
|
|||||||
SET (ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlcleanup")
|
SET (ENGINE_UTILS_DDLCLEANUP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/ddlcleanup")
|
||||||
SET (ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats")
|
SET (ENGINE_UTILS_QUERYSTATS_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/querystats")
|
||||||
SET (ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client")
|
SET (ENGINE_UTILS_LIBMYSQL_CL_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/libmysql_client")
|
||||||
|
SET (ENGINE_UTILS_FDB_WRAPPER "${CMAKE_CURRENT_SOURCE_DIR}/utils/fdb_wrapper_cpp/include")
|
||||||
SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml")
|
SET (ENGINE_WE_CONFIGCPP_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/writeengine/xml")
|
||||||
SET (ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes")
|
SET (ENGINE_DATATYPES_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/datatypes")
|
||||||
SET (ENGINE_BLOCKCACHE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/blockcache")
|
SET (ENGINE_BLOCKCACHE_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/primitives/blockcache")
|
||||||
@ -374,7 +375,7 @@ SET (ENGINE_UTILS_UDFSDK_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/utils/udfsdk"
|
|||||||
|
|
||||||
SET (ENGINE_DEFAULT_INCLUDES ${CMAKE_CURRENT_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR})
|
SET (ENGINE_DEFAULT_INCLUDES ${CMAKE_CURRENT_BINARY_DIR} "." "../" "../../" ${SERVER_BUILD_INCLUDE_DIR})
|
||||||
|
|
||||||
SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE})
|
SET (ENGINE_COMMON_INCLUDES ${ENGINE_DEFAULT_INCLUDES} ${Boost_INCLUDE_DIRS} ${LIBXML2_INCLUDE_DIR} ${ENGINE_UTILS_MESSAGEQCPP_INCLUDE} ${ENGINE_WE_SHARED_INCLUDE} ${ENGINE_UTILS_IDBDATAFILE_INCLUDE} ${ENGINE_UTILS_LOGGINGCPP_INCLUDE} ${ENGINE_UTILS_CONFIGCPP_INCLUDE} ${ENGINE_UTILS_COMPRESS_INCLUDE} ${ENGINE_VERSIONING_BRM_INCLUDE} ${ENGINE_UTILS_ROWGROUP_INCLUDE} ${ENGINE_UTILS_COMMON_INCLUDE} ${ENGINE_UTILS_DATACONVERT_INCLUDE} ${ENGINE_UTILS_RWLOCK_INCLUDE} ${ENGINE_UTILS_FUNCEXP_INCLUDE} ${ENGINE_OAMAPPS_ALARMMANAGER_INCLUDE} ${ENGINE_UTILS_INCLUDE} ${ENGINE_OAM_OAMCPP_INCLUDE} ${ENGINE_DBCON_DDLPKGPROC_INCLUDE} ${ENGINE_DBCON_DDLPKG_INCLUDE} ${ENGINE_DBCON_EXECPLAN_INCLUDE} ${ENGINE_UTILS_STARTUP_INCLUDE} ${ENGINE_DBCON_JOBLIST_INCLUDE} ${ENGINE_WE_WRAPPER_INCLUDE} ${ENGINE_WE_SERVER_INCLUDE} ${ENGINE_DBCON_DMLPKG_INCLUDE} ${ENGINE_WE_CLIENT_INCLUDE} ${ENGINE_DBCON_DMLPKGPROC_INCLUDE} ${ENGINE_UTILS_CACHEUTILS_INCLUDE} ${ENGINE_UTILS_MYSQLCL_INCLUDE} ${ENGINE_UTILS_QUERYTELE_INCLUDE} ${ENGINE_UTILS_THRIFT_INCLUDE} ${ENGINE_UTILS_JOINER_INCLUDE} ${ENGINE_UTILS_THREADPOOL_INCLUDE} ${ENGINE_UTILS_BATCHLDR_INCLUDE} ${ENGINE_UTILS_DDLCLEANUP_INCLUDE} ${ENGINE_UTILS_QUERYSTATS_INCLUDE} ${ENGINE_WE_CONFIGCPP_INCLUDE} ${ENGINE_SERVER_SQL_INCLUDE} ${ENGINE_SERVER_INCLUDE_INCLUDE} ${ENGINE_SERVER_PCRE_INCLUDE} ${ENGINE_SERVER_WSREP_API_INCLUDE} ${ENGINE_SERVER_WSREP_INCLUDE} ${ENGINE_UTILS_UDFSDK_INCLUDE} ${ENGINE_UTILS_LIBMYSQL_CL_INCLUDE} ${ENGINE_DATATYPES_INCLUDE} ${ENGINE_UTILS_FDB_WRAPPER})
|
||||||
|
|
||||||
ADD_SUBDIRECTORY(dbcon/mysql)
|
ADD_SUBDIRECTORY(dbcon/mysql)
|
||||||
IF(NOT TARGET columnstore)
|
IF(NOT TARGET columnstore)
|
||||||
|
@ -212,6 +212,7 @@ clean_old_installation()
|
|||||||
rm -rf /etc/mysql
|
rm -rf /etc/mysql
|
||||||
rm -rf /etc/my.cnf.d/columnstore.cnf
|
rm -rf /etc/my.cnf.d/columnstore.cnf
|
||||||
rm -rf /etc/mysql/mariadb.conf.d/columnstore.cnf
|
rm -rf /etc/mysql/mariadb.conf.d/columnstore.cnf
|
||||||
|
fdbcli --exec "writemode on; clearrange SM_A SM_z"
|
||||||
}
|
}
|
||||||
|
|
||||||
build()
|
build()
|
||||||
|
@ -38,6 +38,8 @@ set(storagemanager_SRCS
|
|||||||
src/SyncTask.cpp
|
src/SyncTask.cpp
|
||||||
src/ListIOTask.cpp
|
src/ListIOTask.cpp
|
||||||
src/TerminateIOTask.cpp
|
src/TerminateIOTask.cpp
|
||||||
|
src/KVStorageInitializer.cpp
|
||||||
|
src/KVPrefixes.cpp
|
||||||
../utils/common/crashtrace.cpp
|
../utils/common/crashtrace.cpp
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -62,13 +64,11 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/storagemanager.cnf.in" "${CMAKE_CURR
|
|||||||
link_directories(${CMAKE_BINARY_DIR}/lib)
|
link_directories(${CMAKE_BINARY_DIR}/lib)
|
||||||
set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib)
|
set(CMAKE_INSTALL_RPATH $ORIGIN $ORIGIN/../lib)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
add_library(storagemanager SHARED ${storagemanager_SRCS})
|
add_library(storagemanager SHARED ${storagemanager_SRCS})
|
||||||
add_dependencies(storagemanager marias3 external_boost)
|
add_dependencies(storagemanager marias3 external_boost fdbcs)
|
||||||
|
|
||||||
target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
|
target_compile_definitions(storagemanager PUBLIC BOOST_NO_CXX11_SCOPED_ENUMS)
|
||||||
target_link_libraries(storagemanager boost_chrono boost_system boost_thread boost_filesystem boost_regex pthread ${S3API_DEPS})
|
target_link_libraries(storagemanager boost_chrono boost_system boost_thread boost_filesystem boost_regex pthread fdbcs ${S3API_DEPS})
|
||||||
|
|
||||||
add_executable(StorageManager src/main.cpp)
|
add_executable(StorageManager src/main.cpp)
|
||||||
target_link_libraries(StorageManager storagemanager)
|
target_link_libraries(StorageManager storagemanager)
|
||||||
|
24
storage-manager/src/KVPrefixes.cpp
Normal file
24
storage-manager/src/KVPrefixes.cpp
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
/* Copyright (C) 2024 MariaDB Corporation
|
||||||
|
|
||||||
|
This program is free software; you can redistribute it and/or
|
||||||
|
modify it under the terms of the GNU General Public License
|
||||||
|
as published by the Free Software Foundation; version 2 of
|
||||||
|
the License.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License
|
||||||
|
along with this program; if not, write to the Free Software
|
||||||
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||||
|
MA 02110-1301, USA. */
|
||||||
|
|
||||||
|
#include "KVPrefixes.hpp"
|
||||||
|
|
||||||
|
namespace storagemanager
|
||||||
|
{
|
||||||
|
// FDB recommends keep the key size up to 32 bytes.
|
||||||
|
const char* KVPrefixes[2] = {/*OWNERSHIP*/ "SM_O_", /*META*/ "SM_M_"};
|
||||||
|
} // namespace storagemanager
|
29
storage-manager/src/KVPrefixes.hpp
Normal file
29
storage-manager/src/KVPrefixes.hpp
Normal file
@ -0,0 +1,29 @@
|
|||||||
|
/* Copyright (C) 2024 MariaDB Corporation
|
||||||
|
|
||||||
|
This program is free software; you can redistribute it and/or
|
||||||
|
modify it under the terms of the GNU General Public License
|
||||||
|
as published by the Free Software Foundation; version 2 of
|
||||||
|
the License.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License
|
||||||
|
along with this program; if not, write to the Free Software
|
||||||
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||||
|
MA 02110-1301, USA. */
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
namespace storagemanager
|
||||||
|
{
|
||||||
|
enum class KVPrefixId
|
||||||
|
{
|
||||||
|
SM_OWNERSHIP = 0,
|
||||||
|
SM_META
|
||||||
|
};
|
||||||
|
|
||||||
|
extern const char* KVPrefixes[];
|
||||||
|
} // namespace storagemanager
|
64
storage-manager/src/KVStorageInitializer.cpp
Normal file
64
storage-manager/src/KVStorageInitializer.cpp
Normal file
@ -0,0 +1,64 @@
|
|||||||
|
/* Copyright (C) 2024 MariaDB Corporation
|
||||||
|
|
||||||
|
This program is free software; you can redistribute it and/or
|
||||||
|
modify it under the terms of the GNU General Public License
|
||||||
|
as published by the Free Software Foundation; version 2 of
|
||||||
|
the License.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License
|
||||||
|
along with this program; if not, write to the Free Software
|
||||||
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||||
|
MA 02110-1301, USA. */
|
||||||
|
#include "Config.h"
|
||||||
|
#include "SMLogging.h"
|
||||||
|
#include "KVStorageInitializer.h"
|
||||||
|
|
||||||
|
namespace storagemanager
|
||||||
|
{
|
||||||
|
static std::shared_ptr<FDBCS::FDBDataBase> fdbDataBaseInstance;
|
||||||
|
static std::unique_ptr<FDBCS::FDBNetwork> fdbNetworkInstance;
|
||||||
|
static std::mutex kvStorageLock;
|
||||||
|
|
||||||
|
static void logError(SMLogging* logger, const char* errorMsg)
|
||||||
|
{
|
||||||
|
logger->log(LOG_CRIT, errorMsg);
|
||||||
|
throw std::runtime_error(errorMsg);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::shared_ptr<FDBCS::FDBDataBase> KVStorageInitializer::getStorageInstance()
|
||||||
|
{
|
||||||
|
if (fdbDataBaseInstance)
|
||||||
|
return fdbDataBaseInstance;
|
||||||
|
|
||||||
|
auto* config = Config::get();
|
||||||
|
auto* logger = SMLogging::get();
|
||||||
|
|
||||||
|
std::unique_lock<std::mutex> lock(kvStorageLock);
|
||||||
|
if (fdbDataBaseInstance)
|
||||||
|
return fdbDataBaseInstance;
|
||||||
|
|
||||||
|
if (!FDBCS::setAPIVersion())
|
||||||
|
logError(logger, "Ownership: FDB setAPIVersion failed.");
|
||||||
|
|
||||||
|
fdbNetworkInstance = std::make_unique<FDBCS::FDBNetwork>();
|
||||||
|
if (!fdbNetworkInstance->setUpAndRunNetwork())
|
||||||
|
logError(logger, "Ownership: FDB setUpAndRunNetwork failed.");
|
||||||
|
|
||||||
|
std::string clusterFilePath = config->getValue("ObjectStorage", "fdb_cluster_file_path");
|
||||||
|
if (clusterFilePath.empty())
|
||||||
|
logError(logger,
|
||||||
|
"Ownership: Need to specify `Foundation DB cluster file path` in the storagemanager.cnf file");
|
||||||
|
|
||||||
|
fdbDataBaseInstance = FDBCS::DataBaseCreator::createDataBase(clusterFilePath);
|
||||||
|
if (!fdbDataBaseInstance)
|
||||||
|
logError(logger, "Ownership: FDB createDataBase failed.");
|
||||||
|
|
||||||
|
return fdbDataBaseInstance;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace storagemanager
|
38
storage-manager/src/KVStorageInitializer.h
Normal file
38
storage-manager/src/KVStorageInitializer.h
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
/* Copyright (C) 2024 MariaDB Corporation
|
||||||
|
|
||||||
|
This program is free software; you can redistribute it and/or
|
||||||
|
modify it under the terms of the GNU General Public License
|
||||||
|
as published by the Free Software Foundation; version 2 of
|
||||||
|
the License.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU General Public License
|
||||||
|
along with this program; if not, write to the Free Software
|
||||||
|
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||||
|
MA 02110-1301, USA. */
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
#include <memory>
|
||||||
|
#include "fdbcs.hpp"
|
||||||
|
|
||||||
|
namespace storagemanager
|
||||||
|
{
|
||||||
|
// Represnts a `Key/Value` storage initializer based on Singleton pattern.
|
||||||
|
// Initializes all needed `Key/Value` storage machinery once and return pointer
|
||||||
|
// to the `Key/Value` storage.
|
||||||
|
class KVStorageInitializer
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
static std::shared_ptr<FDBCS::FDBDataBase> getStorageInstance();
|
||||||
|
KVStorageInitializer() = delete;
|
||||||
|
KVStorageInitializer(const KVStorageInitializer&) = delete;
|
||||||
|
KVStorageInitializer& operator=(KVStorageInitializer&) = delete;
|
||||||
|
KVStorageInitializer(KVStorageInitializer&&) = delete;
|
||||||
|
KVStorageInitializer& operator=(KVStorageInitializer&&) = delete;
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace storagemanager
|
@ -1,4 +1,4 @@
|
|||||||
/* Copyright (C) 2019 MariaDB Corporation
|
/* Copyright (C) 2024 MariaDB Corporation
|
||||||
|
|
||||||
This program is free software; you can redistribute it and/or
|
This program is free software; you can redistribute it and/or
|
||||||
modify it under the terms of the GNU General Public License
|
modify it under the terms of the GNU General Public License
|
||||||
@ -19,6 +19,9 @@
|
|||||||
#include "Config.h"
|
#include "Config.h"
|
||||||
#include "Cache.h"
|
#include "Cache.h"
|
||||||
#include "Synchronizer.h"
|
#include "Synchronizer.h"
|
||||||
|
#include "KVStorageInitializer.h"
|
||||||
|
#include "KVPrefixes.hpp"
|
||||||
|
#include "fdbcs.hpp"
|
||||||
#include <sys/types.h>
|
#include <sys/types.h>
|
||||||
#include <sys/stat.h>
|
#include <sys/stat.h>
|
||||||
#include <unistd.h>
|
#include <unistd.h>
|
||||||
@ -30,6 +33,31 @@ namespace bf = boost::filesystem;
|
|||||||
|
|
||||||
namespace storagemanager
|
namespace storagemanager
|
||||||
{
|
{
|
||||||
|
// FDB recommends keep the key size up to 32 bytes.
|
||||||
|
const char* ownerShipStates[3] = {/*OWNED*/ "_O", /*FLUSHING*/ "_F", /*REQUEST_TRANSFER*/ "_RT"};
|
||||||
|
|
||||||
|
inline std::string getKeyName(const std::string& fileName, OwnershipStateId state)
|
||||||
|
{
|
||||||
|
return KVPrefixes[static_cast<uint32_t>(KVPrefixId::SM_OWNERSHIP)] + fileName +
|
||||||
|
ownerShipStates[static_cast<uint32_t>(state)];
|
||||||
|
}
|
||||||
|
|
||||||
|
inline void Ownership::removeKeys(const std::string& fileName, const std::vector<OwnershipStateId>& states)
|
||||||
|
{
|
||||||
|
auto kvStorage = KVStorageInitializer::getStorageInstance();
|
||||||
|
auto tnx = kvStorage->createTransaction();
|
||||||
|
|
||||||
|
for (const auto state : states)
|
||||||
|
tnx->remove(getKeyName(fileName, state));
|
||||||
|
|
||||||
|
if (!tnx->commit())
|
||||||
|
{
|
||||||
|
const char* msg = "Ownership: commit `removeKeys` failed ";
|
||||||
|
logger->log(LOG_CRIT, msg);
|
||||||
|
throw runtime_error(msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Ownership::Ownership()
|
Ownership::Ownership()
|
||||||
{
|
{
|
||||||
Config* config = Config::get();
|
Config* config = Config::get();
|
||||||
@ -61,6 +89,7 @@ Ownership::Ownership()
|
|||||||
logger->log(LOG_CRIT, msg);
|
logger->log(LOG_CRIT, msg);
|
||||||
throw runtime_error(msg);
|
throw runtime_error(msg);
|
||||||
}
|
}
|
||||||
|
|
||||||
monitor = new Monitor(this);
|
monitor = new Monitor(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -124,28 +153,22 @@ bf::path Ownership::get(const bf::path& p, bool getOwnership)
|
|||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
// minor timesaver
|
|
||||||
#define TOUCH(p, f) \
|
|
||||||
{ \
|
|
||||||
int fd = ::open((metadataPrefix / p / f).string().c_str(), O_TRUNC | O_CREAT | O_WRONLY, 0660); \
|
|
||||||
if (fd >= 0) \
|
|
||||||
::close(fd); \
|
|
||||||
else \
|
|
||||||
{ \
|
|
||||||
char buf[80]; \
|
|
||||||
int saved_errno = errno; \
|
|
||||||
cerr << "failed to touch " << metadataPrefix / p / f << " got " << strerror_r(saved_errno, buf, 80) \
|
|
||||||
<< endl; \
|
|
||||||
} \
|
|
||||||
}
|
|
||||||
|
|
||||||
#define DELETE(p, f) ::unlink((metadataPrefix / p / f).string().c_str());
|
|
||||||
|
|
||||||
void Ownership::touchFlushing(const bf::path& prefix, volatile bool* doneFlushing) const
|
void Ownership::touchFlushing(const bf::path& prefix, volatile bool* doneFlushing) const
|
||||||
{
|
{
|
||||||
while (!*doneFlushing)
|
while (!*doneFlushing)
|
||||||
{
|
{
|
||||||
TOUCH(prefix, "FLUSHING");
|
{
|
||||||
|
auto kvStorage = KVStorageInitializer::getStorageInstance();
|
||||||
|
auto tnx = kvStorage->createTransaction();
|
||||||
|
const std::string flushingKey = getKeyName(prefix.string(), OwnershipStateId::FLUSHING);
|
||||||
|
tnx->set(flushingKey, "");
|
||||||
|
if (!tnx->commit())
|
||||||
|
{
|
||||||
|
const char* msg = "Ownership: commit `touchFlushing` failed ";
|
||||||
|
logger->log(LOG_CRIT, msg);
|
||||||
|
throw runtime_error(msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
boost::this_thread::sleep_for(boost::chrono::seconds(1));
|
boost::this_thread::sleep_for(boost::chrono::seconds(1));
|
||||||
@ -171,10 +194,7 @@ void Ownership::releaseOwnership(const bf::path& p, bool isDtor)
|
|||||||
|
|
||||||
if (isDtor)
|
if (isDtor)
|
||||||
{
|
{
|
||||||
// This is a quick release. If this is being destroyed, then it is through the graceful
|
removeKeys(p.string(), {OwnershipStateId::OWNED, OwnershipStateId::FLUSHING});
|
||||||
// shutdown mechanism, which will flush data separately.
|
|
||||||
DELETE(p, "OWNED");
|
|
||||||
DELETE(p, "FLUSHING");
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
@ -191,19 +211,28 @@ void Ownership::releaseOwnership(const bf::path& p, bool isDtor)
|
|||||||
done = true;
|
done = true;
|
||||||
xfer.interrupt();
|
xfer.interrupt();
|
||||||
xfer.join();
|
xfer.join();
|
||||||
|
removeKeys(p.string(), {OwnershipStateId::OWNED, OwnershipStateId::FLUSHING});
|
||||||
// update state
|
|
||||||
DELETE(p, "OWNED");
|
|
||||||
DELETE(p, "FLUSHING");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
void Ownership::_takeOwnership(const bf::path& p)
|
void Ownership::_takeOwnership(const bf::path& p)
|
||||||
{
|
{
|
||||||
logger->log(LOG_DEBUG, "Ownership: taking ownership of %s", p.string().c_str());
|
logger->log(LOG_DEBUG, "Ownership: taking ownership of %s", p.string().c_str());
|
||||||
DELETE(p, "FLUSHING");
|
{
|
||||||
DELETE(p, "REQUEST_TRANSFER");
|
auto kvStorage = KVStorageInitializer::getStorageInstance();
|
||||||
// TODO: need to consider errors taking ownership
|
auto tnx = kvStorage->createTransaction();
|
||||||
TOUCH(p, "OWNED");
|
const std::string ownedKey = getKeyName(p.string(), OwnershipStateId::OWNED);
|
||||||
|
const std::string flushingKey = getKeyName(p.string(), OwnershipStateId::FLUSHING);
|
||||||
|
const std::string requestTransferKey = getKeyName(p.string(), OwnershipStateId::REQUEST_TRANSFER);
|
||||||
|
tnx->remove(flushingKey);
|
||||||
|
tnx->remove(requestTransferKey);
|
||||||
|
tnx->set(ownedKey, "");
|
||||||
|
if (!tnx->commit())
|
||||||
|
{
|
||||||
|
const char* msg = "Ownership: commit `_takeOwnership` transaction failed";
|
||||||
|
logger->log(LOG_CRIT, msg);
|
||||||
|
throw runtime_error(msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
mutex.lock();
|
mutex.lock();
|
||||||
ownedPrefixes[p] = true;
|
ownedPrefixes[p] = true;
|
||||||
mutex.unlock();
|
mutex.unlock();
|
||||||
@ -225,45 +254,62 @@ void Ownership::takeOwnership(const bf::path& p)
|
|||||||
ownedPrefixes[p] = NULL;
|
ownedPrefixes[p] = NULL;
|
||||||
s.unlock();
|
s.unlock();
|
||||||
|
|
||||||
bool okToTransfer = false;
|
bool ownedKeyExists;
|
||||||
struct stat statbuf;
|
{
|
||||||
int err;
|
auto kvStorage = KVStorageInitializer::getStorageInstance();
|
||||||
char buf[80];
|
auto tnx = kvStorage->createTransaction();
|
||||||
bf::path ownedPath = metadataPrefix / p / "OWNED";
|
const std::string ownedKey = getKeyName(p.string(), OwnershipStateId::OWNED);
|
||||||
bf::path flushingPath = metadataPrefix / p / "FLUSHING";
|
ownedKeyExists = tnx->get(ownedKey).first;
|
||||||
|
}
|
||||||
|
|
||||||
// if it's not already owned, then we can take possession
|
// if it's not already owned, then we can take possession
|
||||||
err = ::stat(ownedPath.string().c_str(), &statbuf);
|
if (!ownedKeyExists)
|
||||||
if (err && errno == ENOENT)
|
|
||||||
{
|
{
|
||||||
_takeOwnership(p);
|
_takeOwnership(p);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
TOUCH(p, "REQUEST_TRANSFER");
|
{
|
||||||
|
auto kvStorage = KVStorageInitializer::getStorageInstance();
|
||||||
|
auto tnx = kvStorage->createTransaction();
|
||||||
|
const std::string requestTransferKey = getKeyName(p.string(), OwnershipStateId::REQUEST_TRANSFER);
|
||||||
|
tnx->set(requestTransferKey, "");
|
||||||
|
if (!tnx->commit())
|
||||||
|
{
|
||||||
|
const char* msg = "Ownership: commit `requestTransfer` failed ";
|
||||||
|
logger->log(LOG_CRIT, msg);
|
||||||
|
throw runtime_error(msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool okToTransfer = false;
|
||||||
time_t lastFlushTime = time(NULL);
|
time_t lastFlushTime = time(NULL);
|
||||||
while (!okToTransfer && time(NULL) < lastFlushTime + 10)
|
while (!okToTransfer && time(NULL) < lastFlushTime + 10)
|
||||||
{
|
{
|
||||||
// if the OWNED file is deleted or if the flushing file isn't touched after 10 secs
|
// if the OWNED file is deleted or if the flushing file isn't touched after 10 secs
|
||||||
// it is ok to take possession.
|
// it is ok to take possession.
|
||||||
err = ::stat(ownedPath.string().c_str(), &statbuf);
|
bool ownedKeyExists;
|
||||||
if (err)
|
|
||||||
{
|
{
|
||||||
if (errno == ENOENT)
|
auto kvStorage = KVStorageInitializer::getStorageInstance();
|
||||||
okToTransfer = true;
|
auto tnx = kvStorage->createTransaction();
|
||||||
else
|
const std::string ownedKey = getKeyName(p.string(), OwnershipStateId::OWNED);
|
||||||
logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s",
|
ownedKeyExists = tnx->get(ownedKey).first;
|
||||||
strerror_r(errno, buf, 80), ownedPath.string().c_str());
|
|
||||||
}
|
}
|
||||||
err = ::stat(flushingPath.string().c_str(), &statbuf);
|
if (!ownedKeyExists)
|
||||||
if (err && errno != ENOENT)
|
okToTransfer = true;
|
||||||
logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s",
|
|
||||||
strerror_r(errno, buf, 80), flushingPath.string().c_str());
|
bool flushingKeyExists;
|
||||||
else
|
{
|
||||||
|
auto kvStorage = KVStorageInitializer::getStorageInstance();
|
||||||
|
auto tnx = kvStorage->createTransaction();
|
||||||
|
const std::string flushingKey = getKeyName(p.string(), OwnershipStateId::FLUSHING);
|
||||||
|
flushingKeyExists = tnx->get(flushingKey).first;
|
||||||
|
}
|
||||||
|
if (flushingKeyExists)
|
||||||
{
|
{
|
||||||
logger->log(LOG_DEBUG, "Ownership: waiting to get %s", p.string().c_str());
|
logger->log(LOG_DEBUG, "Ownership: waiting to get %s", p.string().c_str());
|
||||||
if (!err)
|
// Since notice the flushing key.
|
||||||
lastFlushTime = statbuf.st_mtime;
|
lastFlushTime = time(NULL);
|
||||||
}
|
}
|
||||||
if (!okToTransfer)
|
if (!okToTransfer)
|
||||||
sleep(1);
|
sleep(1);
|
||||||
@ -286,9 +332,6 @@ Ownership::Monitor::~Monitor()
|
|||||||
void Ownership::Monitor::watchForInterlopers()
|
void Ownership::Monitor::watchForInterlopers()
|
||||||
{
|
{
|
||||||
// look for requests to transfer ownership
|
// look for requests to transfer ownership
|
||||||
struct stat statbuf;
|
|
||||||
int err;
|
|
||||||
char buf[80];
|
|
||||||
vector<bf::path> releaseList;
|
vector<bf::path> releaseList;
|
||||||
|
|
||||||
while (!stop)
|
while (!stop)
|
||||||
@ -302,17 +345,20 @@ void Ownership::Monitor::watchForInterlopers()
|
|||||||
break;
|
break;
|
||||||
if (prefix.second == false)
|
if (prefix.second == false)
|
||||||
continue;
|
continue;
|
||||||
bf::path p(owner->metadataPrefix / (prefix.first) / "REQUEST_TRANSFER");
|
|
||||||
const char* cp = p.string().c_str();
|
|
||||||
|
|
||||||
err = ::stat(cp, &statbuf);
|
bool requestKeyExists;
|
||||||
|
{
|
||||||
|
auto kvStorage = KVStorageInitializer::getStorageInstance();
|
||||||
|
auto tnx = kvStorage->createTransaction();
|
||||||
|
const std::string requestTransferKey =
|
||||||
|
getKeyName(prefix.first.string(), OwnershipStateId::REQUEST_TRANSFER);
|
||||||
|
|
||||||
|
requestKeyExists = tnx->get(requestTransferKey).first;
|
||||||
|
}
|
||||||
// release it if there's a release request only. Log it if there's an error other than
|
// release it if there's a release request only. Log it if there's an error other than
|
||||||
// that the file isn't there.
|
// that the file isn't there.
|
||||||
if (err == 0)
|
if (requestKeyExists)
|
||||||
releaseList.push_back(prefix.first);
|
releaseList.push_back(prefix.first);
|
||||||
if (err < 0 && errno != ENOENT)
|
|
||||||
owner->logger->log(LOG_ERR, "Runner::watchForInterlopers(): failed to stat %s, got %s", cp,
|
|
||||||
strerror_r(errno, buf, 80));
|
|
||||||
}
|
}
|
||||||
s.unlock();
|
s.unlock();
|
||||||
|
|
||||||
@ -329,5 +375,4 @@ void Ownership::Monitor::watchForInterlopers()
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} // namespace storagemanager
|
||||||
} // namespace storagemanager
|
|
@ -22,12 +22,20 @@
|
|||||||
#include <boost/thread.hpp>
|
#include <boost/thread.hpp>
|
||||||
#include <map>
|
#include <map>
|
||||||
#include "SMLogging.h"
|
#include "SMLogging.h"
|
||||||
|
#include "fdbcs.hpp"
|
||||||
|
|
||||||
/* This class tracks the ownership of each prefix and manages ownership transfer.
|
/* This class tracks the ownership of each prefix and manages ownership transfer.
|
||||||
Could we come up with a better name btw? */
|
Could we come up with a better name btw? */
|
||||||
|
|
||||||
namespace storagemanager
|
namespace storagemanager
|
||||||
{
|
{
|
||||||
|
enum class OwnershipStateId
|
||||||
|
{
|
||||||
|
OWNED = 0,
|
||||||
|
FLUSHING,
|
||||||
|
REQUEST_TRANSFER
|
||||||
|
};
|
||||||
|
|
||||||
class Ownership : public boost::noncopyable
|
class Ownership : public boost::noncopyable
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
@ -50,6 +58,7 @@ class Ownership : public boost::noncopyable
|
|||||||
void takeOwnership(const boost::filesystem::path&);
|
void takeOwnership(const boost::filesystem::path&);
|
||||||
void releaseOwnership(const boost::filesystem::path&, bool isDtor = false);
|
void releaseOwnership(const boost::filesystem::path&, bool isDtor = false);
|
||||||
void _takeOwnership(const boost::filesystem::path&);
|
void _takeOwnership(const boost::filesystem::path&);
|
||||||
|
inline void removeKeys(const std::string& fileName, const std::vector<OwnershipStateId>& states);
|
||||||
|
|
||||||
struct Monitor
|
struct Monitor
|
||||||
{
|
{
|
||||||
|
@ -80,6 +80,10 @@ max_concurrent_uploads = 21
|
|||||||
# negotiate ownership of data from a failed instance.
|
# negotiate ownership of data from a failed instance.
|
||||||
common_prefix_depth = 3
|
common_prefix_depth = 3
|
||||||
|
|
||||||
|
# This value is used to specify a cluster file path to FDB cluster.
|
||||||
|
# See more: https://apple.github.io/foundationdb/administration.html
|
||||||
|
fdb_cluster_file_path = /etc/foundationdb/fdb.cluster
|
||||||
|
|
||||||
[S3]
|
[S3]
|
||||||
# These should be self-explanatory. Region can be blank or commented
|
# These should be self-explanatory. Region can be blank or commented
|
||||||
# if using a private cloud storage system. Bucket has to be set to
|
# if using a private cloud storage system. Bucket has to be set to
|
||||||
|
@ -16,7 +16,7 @@
|
|||||||
MA 02110-1301, USA. */
|
MA 02110-1301, USA. */
|
||||||
|
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include "../include/fdbcs.hpp"
|
#include "fdbcs.hpp"
|
||||||
|
|
||||||
using namespace std;
|
using namespace std;
|
||||||
using namespace FDBCS;
|
using namespace FDBCS;
|
||||||
|
Reference in New Issue
Block a user