diff --git a/CMakeLists.txt b/CMakeLists.txt index afdcdb2f2..c627c5d01 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,15 +1,6 @@ cmake_minimum_required(VERSION 2.8.12) project(storagemanager) -option(ENABLE_UNITY_BUILD "" ON) -option(BUILD_SHARED_LIBS "" OFF) -option(ENABLE_TESTING "" OFF) -set(CMAKE_INSTALL_PREFIX_ORIG ${CMAKE_INSTALL_PREFIX}) -set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/aws-sdk-deps) -set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp) -add_subdirectory(aws-sdk-cpp) -set(CMAKE_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX_ORIG}) - include_directories(include) set(storagemanager_SRCS @@ -29,6 +20,9 @@ set(storagemanager_SRCS src/CopyTask.cpp src/IOCoordinator.cpp src/SessionManager.cpp + src/Config.cpp + src/CloudStorage.cpp + src/S3Storage.cpp ) option(TRACE "Enable some tracing output" OFF) @@ -42,6 +36,16 @@ target_link_libraries(StorageManager boost_system boost_thread boost_filesystem add_executable(unit_tests src/unit_tests.cpp ${storagemanager_SRCS}) target_link_libraries(unit_tests boost_system boost_thread boost_filesystem) +# config & import the AWS SDK +option(ENABLE_UNITY_BUILD "" ON) +option(BUILD_SHARED_LIBS "" OFF) +option(ENABLE_TESTING "" OFF) +set(CMAKE_INSTALL_PREFIX_ORIG ${CMAKE_INSTALL_PREFIX}) +set(CMAKE_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/aws-sdk-deps) +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_CURRENT_SOURCE_DIR}/aws-sdk-cpp) +add_subdirectory(aws-sdk-cpp) +set(CMAKE_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX_ORIG}) + #install(TARGETS StorageManager DESTINATION ${ENGINE_BINDIR} COMPONENT platform) diff --git a/src/CloudStorage.cpp b/src/CloudStorage.cpp new file mode 100644 index 000000000..ed01a6c01 --- /dev/null +++ b/src/CloudStorage.cpp @@ -0,0 +1,39 @@ + +#include "CloudStorage.h" +#include "Config.h" +#include "S3Storage.h" +#include +#include +#include + +using namespace std; + +namespace +{ + boost::mutex m; + storagemanager::CloudStorage *inst; +} + +namespace storagemanager +{ +CloudStorage * CloudStorage::get() +{ + if (inst) + return inst; + + Config *conf = Config::get(); + string type = conf->getValue("ObjectStorage", "service"); + boost::mutex::scoped_lock s(m); + if (inst) + return inst; + if (type == "s3" || type == "S3") + inst = new S3Storage(); + else { + syslog(LOG_CRIT, "CloudStorage: got unknown service provider: %s", type.c_str()); + return NULL; + } + + return inst; +} + +} diff --git a/src/CloudStorage.h b/src/CloudStorage.h new file mode 100644 index 000000000..174b31888 --- /dev/null +++ b/src/CloudStorage.h @@ -0,0 +1,24 @@ + +#ifndef CLOUDSTORAGE_H_ +#define CLOUDSTORAGE_H_ + +#include + +namespace storagemanager +{ + +class CloudStorage +{ + public: + virtual int getObject(const std::string &sourceKey, const std::string &destFile) = 0; + virtual int putObject(const std::string &sourceFile, const std::string &destKey) = 0; + virtual void deleteObject(const std::string &key) = 0; + + // this will return a CloudStorage instance of the type specified in StorageManager.cnf + static CloudStorage *get(); + private: +}; + +} + +#endif diff --git a/src/Config.cpp b/src/Config.cpp new file mode 100644 index 000000000..c4484c721 --- /dev/null +++ b/src/Config.cpp @@ -0,0 +1,70 @@ + +#include "Config.h" +#include +#include +#include + +#include +#include + +using namespace std; + +namespace +{ + boost::mutex m; + storagemanager::Config *inst = NULL; +} + +namespace storagemanager +{ + +Config * Config::get() +{ + if (inst) + return inst; + boost::mutex::scoped_lock s(m); + if (inst) + return inst; + inst = new Config(); + return inst; +} + +Config::Config() +{ + /* This will search the current directory, + then the $COLUMNSTORE_INSTALL_DIR/etc, + then /etc + looking for storagemanager.cnf + + We can change this however we need later. + */ + char *cs_install_dir = getenv("COLUMNSTORE_INSTALL_DIR"); + + vector paths; + + // the paths to search in order + paths.push_back("./"); + if (cs_install_dir) + paths.push_back(cs_install_dir); + paths.push_back("/etc"); + + for (int i = 0; i < paths.size(); i++) + { + if (boost::filesystem::exists(paths[i] + "/storagemanager.cnf")) + { + filename = paths[i] + "/storagemanager.cnf"; + break; + } + } + if (filename.empty()) + throw runtime_error("Config: Could not find the config file for StorageManager"); + + boost::property_tree::ini_parser::read_ini(filename, contents); +} + +string Config::getValue(const string §ion, const string &key) const +{ + return contents.get(section + "." + key); +} + +} diff --git a/src/Config.h b/src/Config.h new file mode 100644 index 000000000..56e185f38 --- /dev/null +++ b/src/Config.h @@ -0,0 +1,27 @@ + +#ifndef CONFIG_H_ +#define CONFIG_H_ + +#include +#include + + +namespace storagemanager +{ +class Config : public boost::noncopyable +{ + public: + static Config *get(); + + std::string getValue(const std::string §ion, const std::string &key) const; + + private: + Config(); + + std::string filename; + boost::property_tree::ptree contents; +}; + +} + +#endif diff --git a/src/S3Storage.cpp b/src/S3Storage.cpp new file mode 100644 index 000000000..e3c001243 --- /dev/null +++ b/src/S3Storage.cpp @@ -0,0 +1,31 @@ + +#include "S3Storage.h" + +using namespace std; + +namespace storagemanager +{ + +S3Storage::S3Storage() +{ +} + +S3Storage::~S3Storage() +{ +} + +int S3Storage::getObject(const string &courceKey, const string &destFile) +{ + return 0; +} + +int S3Storage::putObject(const string &sourceFile, const string &destKey) +{ + return 0; +} + +void S3Storage::deleteObject(const string &key) +{ +} + +} diff --git a/src/S3Storage.h b/src/S3Storage.h new file mode 100644 index 000000000..48066d5c7 --- /dev/null +++ b/src/S3Storage.h @@ -0,0 +1,30 @@ + +#ifndef S3STORAGE_H_ +#define S3STORAGE_H_ + +#include +#include "CloudStorage.h" + +namespace storagemanager +{ + +class S3Storage : public CloudStorage +{ + public: + S3Storage(); + virtual ~S3Storage(); + + int getObject(const std::string &sourceKey, const std::string &destFile); + int putObject(const std::string &sourceFile, const std::string &destKey); + void deleteObject(const std::string &key); + + private: + +}; + + + + +} + +#endif diff --git a/storagemanager.cnf b/storagemanager.cnf new file mode 100644 index 000000000..ee2e186a8 --- /dev/null +++ b/storagemanager.cnf @@ -0,0 +1,10 @@ +[ObjectStorage] +service=S3 + +[S3] +region = us-west-1 + +# S3 credentials will come through standard AWS methods +# ex, by setting AWS_ACCESS_KEY_ID & AWS_SECRET_ACCESS_KEY envvars. +# see https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html +