Made the cloud IO lib a plugin to the IDB filesystem stuff.

It loads correctly.

utils/cloudio/CMakeLists.txt

@@ -1,13 +1,13 @@
 include_directories(${ENGINE_COMMON_INCLUDES} storage-manager/include)
 
-set(cloudio_LIB_SRCS SMComm.cpp SMDataFile.cpp SMFileFactory.cpp SMFileSystem.cpp SocketPool.cpp)
+set(cloudio_LIB_SRCS SMComm.cpp SMDataFile.cpp SMFileFactory.cpp SMFileSystem.cpp SocketPool.cpp cloud_plugin.cpp)
 
 add_library(cloudio SHARED ${cloudio_LIB_SRCS})
 
 # IDBDataFile currently depends on cloudio, which is backward.
 # Once cloudio has been turned into a proper plugin for idbdatafile,
 # we should be able to reverse the dependency like so:
-# target_link_libraries(cloudio idbdatafile messageqcpp loggingcpp)
+target_link_libraries(cloudio idbdatafile messageqcpp loggingcpp)
 
 set_target_properties(cloudio PROPERTIES VERSION 1.0.0 SOVERSION 1)
 
@@ -21,5 +21,6 @@ add_executable(cloudio_component_test component_test.cpp)
 # untangle all of that and declare lib dependencies properly.
 # For now I'm going to do like the other executables, which means
 # nearly everything AFAICT.
-target_link_libraries(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS})
+target_link_libraries(cloudio_component_test ${ENGINE_LDFLAGS} ${ENGINE_EXEC_LIBS} ${MARIADB_CLIENT_LIBS} cloudio)
+#target_link_libraries(cloudio_component_test cloudio)
 

utils/idbdatafile/CMakeLists.txt

@@ -15,7 +15,7 @@ set(idbdatafile_LIB_SRCS
 
 add_library(idbdatafile SHARED ${idbdatafile_LIB_SRCS})
 
-target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES} cloudio)
+target_link_libraries(idbdatafile ${NETSNMP_LIBRARIES})
 
 set_target_properties(idbdatafile PROPERTIES VERSION 1.0.0 SOVERSION 1)
 

utils/idbdatafile/IDBFactory.cpp

@@ -55,7 +55,7 @@ bool IDBFactory::installDefaultPlugins()
     
     // TODO: use the installPlugin fcn below instead of declaring this statically, then remove the dependency
     // IDBDatafile -> cloudio
-    s_plugins[IDBDataFile::CLOUD] = FileFactoryEnt(IDBDataFile::CLOUD, "cloud", new SMFileFactory(), new SMFileSystem());
+    //s_plugins[IDBDataFile::CLOUD] = FileFactoryEnt(IDBDataFile::CLOUD, "cloud", new SMFileFactory(), new SMFileSystem());
     
     return false;
 }
@@ -100,6 +100,14 @@ bool IDBFactory::installPlugin(const std::string& plugin)
 #endif
 }
 
+vector<IDBDataFile::Types> IDBFactory::listPlugins()
+{
+    vector<IDBDataFile::Types> ret;
+    for (FactoryMap::iterator it = s_plugins.begin(); it != s_plugins.end(); ++it)
+        ret.push_back(it->first);
+    return ret;
+}
+
 IDBDataFile* IDBFactory::open(IDBDataFile::Types type, const char* fname, const char* mode, unsigned opts, unsigned colWidth)
 {
     if ( s_plugins.find(type) == s_plugins.end() )

utils/idbdatafile/IDBFactory.h

@@ -20,6 +20,7 @@
 
 #include <string>
 #include <map>
+#include <vector>
 #include <stdexcept>
 
 #include "IDBDataFile.h"
@@ -80,6 +81,11 @@ public:
      */
     static bool installPlugin(const std::string& plugin);
 
+    /**
+     * This method lists the loaded plugins by type.
+     */
+    static std::vector<IDBDataFile::Types> listPlugins();
+    
     /**
      * This method calls the Factory for the specified type
      */
@@ -94,7 +100,7 @@ public:
      * This retrieves the IDBFileSystem for the specified type
      */
     static const std::string& name(IDBDataFile::Types type);
-
+    
 private:
     typedef std::map<IDBDataFile::Types, FileFactoryEnt> FactoryMap;
     typedef FactoryMap::const_iterator FactoryMapCIter;

utils/idbdatafile/IDBPolicy.cpp

@@ -43,6 +43,7 @@ namespace idbdatafile
 {
 
 bool IDBPolicy::s_usehdfs = false;
+bool IDBPolicy::s_usecloud = false;
 bool IDBPolicy::s_bUseRdwrMemBuffer = false;
 int64_t IDBPolicy::s_hdfsRdwrBufferMaxSize = 0;
 std::string IDBPolicy::s_hdfsRdwrScratch;
@@ -98,20 +99,12 @@ bool IDBPolicy::installPlugin(const std::string& plugin)
 {
     bool ret = IDBFactory::installPlugin(plugin);
 
-    // this is a cheesy way to do this, but it seems as good as anything for
+    vector<IDBDataFile::Types> plugins = IDBFactory::listPlugins();
-    // now.  At some point, this policy class needs to be data driven - some
+    for (uint i = 0; i < plugins.size(); i++)
-    // type of specification to drive the logic here.
+        if (plugins[i] == IDBDataFile::HDFS)
-    try
+            s_usehdfs = true;
-    {
+        else if (plugins[i] == IDBDataFile::CLOUD)
-        // see if there is an HDFS plugin
+            s_usecloud = true;
-        IDBFactory::name(IDBDataFile::HDFS);
-        s_usehdfs = true;
-    }
-    catch (std::exception& )
-    {
-        // nothing to do - this just means the plugin was not HDFS
-        ;
-    }
 
     return ret;
 }
@@ -130,16 +123,27 @@ bool IDBPolicy::isLocalFile( const std::string& path )
     bool isXml = (fileExt == ".xml");
 
     bool isVb = path.find("versionbuffer") != string::npos;
-    bool isInDbroot = path.find("/Calpont/data") != string::npos;
     bool isScratch = path.find(s_hdfsRdwrScratch) == 0;
 
-    return !isInDbroot || isXml || isVb || isScratch;
+    return isXml || isVb || isScratch;
 }
 
 IDBDataFile::Types IDBPolicy::getType( const std::string& path, Contexts ctxt )
 {
     bool isLocal = isLocalFile( path );
 
+    if (isLocal)
+        if (ctxt == PRIMPROC)
+            return IDBDataFile::UNBUFFERED;
+        else
+            return IDBDataFile::BUFFERED;
+    else if (useHdfs())
+        return IDBDataFile::HDFS;
+    else if (useCloud())
+        return IDBDataFile::CLOUD;
+    throw runtime_error("IDBPolicy: No appropriate data file type");
+    
+#if 0
     if ( ctxt == PRIMPROC )
     {
         if ( isLocal || !useHdfs() )
@@ -154,6 +158,7 @@ IDBDataFile::Types IDBPolicy::getType( const std::string& path, Contexts ctxt )
         else
             return IDBDataFile::HDFS;
     }
+#endif
 }
 
 IDBFileSystem& IDBPolicy::getFs( const std::string& path )

utils/idbdatafile/IDBPolicy.h

@@ -80,6 +80,11 @@ public:
      */
     static bool useHdfs();
 
+    /**
+     * Accessor method that returns whether or not cloud IO is enabled
+     */
+    static bool useCloud();
+    
     /**
      * Accessor method that returns whether to use HDFS memory buffers
      */
@@ -134,6 +139,7 @@ private:
     static bool isLocalFile( const std::string& path );
 
     static bool s_usehdfs;
+    static bool s_usecloud;
     static bool s_bUseRdwrMemBuffer;
     static std::string s_hdfsRdwrScratch;
     static int64_t s_hdfsRdwrBufferMaxSize;
@@ -153,6 +159,12 @@ bool IDBPolicy::useHdfs()
     return s_usehdfs;
 }
 
+inline
+bool IDBPolicy::useCloud()
+{
+    return s_usecloud;
+}
+
 inline
 bool IDBPolicy::useRdwrMemBuffer()
 {