From d97a570b866a5ad3c0d6a74d8178318e030101ec Mon Sep 17 00:00:00 2001 From: Patrick LeBlanc Date: Mon, 4 Feb 2019 10:41:21 -0600 Subject: [PATCH] Made the cloud IO lib a plugin to the IDB filesystem stuff. It loads correctly. --- utils/cloudio/CMakeLists.txt | 7 +++--- utils/idbdatafile/CMakeLists.txt | 2 +- utils/idbdatafile/IDBFactory.cpp | 10 ++++++++- utils/idbdatafile/IDBFactory.h | 8 ++++++- utils/idbdatafile/IDBPolicy.cpp | 37 ++++++++++++++++++-------------- utils/idbdatafile/IDBPolicy.h | 12 +++++++++++ 6 files changed, 54 insertions(+), 22 deletions(-) diff --git a/utils/cloudio/CMakeLists.txt b/utils/cloudio/CMakeLists.txt index 7e6ec1c25..03375e68d 100644 --- a/utils/cloudio/CMakeLists.txt +++ b/utils/cloudio/CMakeLists.txt @@ -1,13 +1,13 @@ include_directories(${ENGINE_COMMON_INCLUDES} storage-manager/include) -set(cloudio_LIB_SRCS SMComm.cpp SMDataFile.cpp SMFileFactory.cpp SMFileSystem.cpp SocketPool.cpp) +set(cloudio_LIB_SRCS SMComm.cpp SMDataFile.cpp SMFileFactory.cpp SMFileSystem.cpp SocketPool.cpp cloud_plugin.cpp) add_library(cloudio SHARED ${cloudio_LIB_SRCS}) # IDBDataFile currently depends on cloudio, which is backward. # Once cloudio has been turned into a proper plugin for idbdatafile, # we should be able to reverse the dependency like so: -# target_link_libraries(cloudio idbdatafile messageqcpp loggingcpp) +target_link_libraries(cloudio idbdatafile messageqcpp loggingcpp) set_target_properties(cloudio PROPERTIES VERSION 1.0.0 SOVERSION 1) @@ -21,5 +21,6 @@ add_executable(cloudio_component_test component_test.cpp) # untangle all of that and declare lib dependencies properly. # For now I'm going to do like the other executables, which means # nearly everything AFAICT. -target_link_libraries(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS}) +target_link_libraries(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS} cloudio) +#target_link_libraries(cloudio_component_test cloudio) diff --git a/utils/idbdatafile/CMakeLists.txt b/utils/idbdatafile/CMakeLists.txt index cf809ffec..7a39665aa 100644 --- a/utils/idbdatafile/CMakeLists.txt +++ b/utils/idbdatafile/CMakeLists.txt @@ -15,7 +15,7 @@ set(idbdatafile_LIB_SRCS add_library(idbdatafile SHARED ${idbdatafile_LIB_SRCS}) -target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES} cloudio) +target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES}) set_target_properties(idbdatafile PROPERTIES VERSION 1.0.0 SOVERSION 1) diff --git a/utils/idbdatafile/IDBFactory.cpp b/utils/idbdatafile/IDBFactory.cpp index e1fbf1da9..1956dbf61 100644 --- a/utils/idbdatafile/IDBFactory.cpp +++ b/utils/idbdatafile/IDBFactory.cpp @@ -55,7 +55,7 @@ bool IDBFactory::installDefaultPlugins() // TODO: use the installPlugin fcn below instead of declaring this statically, then remove the dependency // IDBDatafile -> cloudio - s_plugins[IDBDataFile::CLOUD] = FileFactoryEnt(IDBDataFile::CLOUD, "cloud", new SMFileFactory(), new SMFileSystem()); + //s_plugins[IDBDataFile::CLOUD] = FileFactoryEnt(IDBDataFile::CLOUD, "cloud", new SMFileFactory(), new SMFileSystem()); return false; } @@ -100,6 +100,14 @@ bool IDBFactory::installPlugin(const std::string& plugin) #endif } +vector IDBFactory::listPlugins() +{ + vector ret; + for (FactoryMap::iterator it = s_plugins.begin(); it != s_plugins.end(); ++it) + ret.push_back(it->first); + return ret; +} + IDBDataFile* IDBFactory::open(IDBDataFile::Types type, const char* fname, const char* mode, unsigned opts, unsigned colWidth) { if ( s_plugins.find(type) == s_plugins.end() ) diff --git a/utils/idbdatafile/IDBFactory.h b/utils/idbdatafile/IDBFactory.h index 8ad5239b6..83e29e0a8 100644 --- a/utils/idbdatafile/IDBFactory.h +++ b/utils/idbdatafile/IDBFactory.h @@ -20,6 +20,7 @@ #include #include +#include #include #include "IDBDataFile.h" @@ -80,6 +81,11 @@ public: */ static bool installPlugin(const std::string& plugin); + /** + * This method lists the loaded plugins by type. + */ + static std::vector listPlugins(); + /** * This method calls the Factory for the specified type */ @@ -94,7 +100,7 @@ public: * This retrieves the IDBFileSystem for the specified type */ static const std::string& name(IDBDataFile::Types type); - + private: typedef std::map FactoryMap; typedef FactoryMap::const_iterator FactoryMapCIter; diff --git a/utils/idbdatafile/IDBPolicy.cpp b/utils/idbdatafile/IDBPolicy.cpp index 4d0679308..9264fa784 100644 --- a/utils/idbdatafile/IDBPolicy.cpp +++ b/utils/idbdatafile/IDBPolicy.cpp @@ -43,6 +43,7 @@ namespace idbdatafile { bool IDBPolicy::s_usehdfs = false; +bool IDBPolicy::s_usecloud = false; bool IDBPolicy::s_bUseRdwrMemBuffer = false; int64_t IDBPolicy::s_hdfsRdwrBufferMaxSize = 0; std::string IDBPolicy::s_hdfsRdwrScratch; @@ -98,20 +99,12 @@ bool IDBPolicy::installPlugin(const std::string& plugin) { bool ret = IDBFactory::installPlugin(plugin); - // this is a cheesy way to do this, but it seems as good as anything for - // now. At some point, this policy class needs to be data driven - some - // type of specification to drive the logic here. - try - { - // see if there is an HDFS plugin - IDBFactory::name(IDBDataFile::HDFS); - s_usehdfs = true; - } - catch (std::exception& ) - { - // nothing to do - this just means the plugin was not HDFS - ; - } + vector plugins = IDBFactory::listPlugins(); + for (uint i = 0; i < plugins.size(); i++) + if (plugins[i] == IDBDataFile::HDFS) + s_usehdfs = true; + else if (plugins[i] == IDBDataFile::CLOUD) + s_usecloud = true; return ret; } @@ -130,16 +123,27 @@ bool IDBPolicy::isLocalFile( const std::string& path ) bool isXml = (fileExt == ".xml"); bool isVb = path.find("versionbuffer") != string::npos; - bool isInDbroot = path.find("/Calpont/data") != string::npos; bool isScratch = path.find(s_hdfsRdwrScratch) == 0; - return !isInDbroot || isXml || isVb || isScratch; + return isXml || isVb || isScratch; } IDBDataFile::Types IDBPolicy::getType( const std::string& path, Contexts ctxt ) { bool isLocal = isLocalFile( path ); + if (isLocal) + if (ctxt == PRIMPROC) + return IDBDataFile::UNBUFFERED; + else + return IDBDataFile::BUFFERED; + else if (useHdfs()) + return IDBDataFile::HDFS; + else if (useCloud()) + return IDBDataFile::CLOUD; + throw runtime_error("IDBPolicy: No appropriate data file type"); + +#if 0 if ( ctxt == PRIMPROC ) { if ( isLocal || !useHdfs() ) @@ -154,6 +158,7 @@ IDBDataFile::Types IDBPolicy::getType( const std::string& path, Contexts ctxt ) else return IDBDataFile::HDFS; } +#endif } IDBFileSystem& IDBPolicy::getFs( const std::string& path ) diff --git a/utils/idbdatafile/IDBPolicy.h b/utils/idbdatafile/IDBPolicy.h index 333b71808..08fecb0e3 100644 --- a/utils/idbdatafile/IDBPolicy.h +++ b/utils/idbdatafile/IDBPolicy.h @@ -80,6 +80,11 @@ public: */ static bool useHdfs(); + /** + * Accessor method that returns whether or not cloud IO is enabled + */ + static bool useCloud(); + /** * Accessor method that returns whether to use HDFS memory buffers */ @@ -134,6 +139,7 @@ private: static bool isLocalFile( const std::string& path ); static bool s_usehdfs; + static bool s_usecloud; static bool s_bUseRdwrMemBuffer; static std::string s_hdfsRdwrScratch; static int64_t s_hdfsRdwrBufferMaxSize; @@ -153,6 +159,12 @@ bool IDBPolicy::useHdfs() return s_usehdfs; } +inline +bool IDBPolicy::useCloud() +{ + return s_usecloud; +} + inline bool IDBPolicy::useRdwrMemBuffer() {