diff --git a/CMakeLists.txt b/CMakeLists.txt index c627c5d01..4b61d43c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ set(storagemanager_SRCS src/Config.cpp src/CloudStorage.cpp src/S3Storage.cpp + src/LocalStorage.cpp ) option(TRACE "Enable some tracing output" OFF) @@ -31,10 +32,10 @@ if (TRACE) endif() add_executable(StorageManager src/main.cpp ${storagemanager_SRCS}) -target_link_libraries(StorageManager boost_system boost_thread boost_filesystem aws-cpp-sdk-s3) +target_link_libraries(StorageManager boost_system boost_thread boost_filesystem boost_regex aws-cpp-sdk-s3) add_executable(unit_tests src/unit_tests.cpp ${storagemanager_SRCS}) -target_link_libraries(unit_tests boost_system boost_thread boost_filesystem) +target_link_libraries(unit_tests boost_system boost_thread boost_filesystem boost_regex) # config & import the AWS SDK option(ENABLE_UNITY_BUILD "" ON) diff --git a/src/CloudStorage.h b/src/CloudStorage.h index 174b31888..14d8924e6 100644 --- a/src/CloudStorage.h +++ b/src/CloudStorage.h @@ -13,6 +13,7 @@ class CloudStorage virtual int getObject(const std::string &sourceKey, const std::string &destFile) = 0; virtual int putObject(const std::string &sourceFile, const std::string &destKey) = 0; virtual void deleteObject(const std::string &key) = 0; + virtual int copyObject(const std::string &sourceKey, const std::string &destKey) = 0; // this will return a CloudStorage instance of the type specified in StorageManager.cnf static CloudStorage *get(); diff --git a/src/Config.cpp b/src/Config.cpp index c4484c721..7ba674370 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -32,7 +33,7 @@ Config * Config::get() Config::Config() { /* This will search the current directory, - then the $COLUMNSTORE_INSTALL_DIR/etc, + then $COLUMNSTORE_INSTALL_DIR/etc, then /etc looking for storagemanager.cnf @@ -43,7 +44,7 @@ Config::Config() vector paths; // the paths to search in order - paths.push_back("./"); + paths.push_back("."); if (cs_install_dir) paths.push_back(cs_install_dir); paths.push_back("/etc"); @@ -62,9 +63,20 @@ Config::Config() boost::property_tree::ini_parser::read_ini(filename, contents); } +string use_envvar(boost::smatch envvar) +{ + char *env = getenv(envvar[1].str().c_str()); + return (env ? env : ""); +} + string Config::getValue(const string §ion, const string &key) const { - return contents.get(section + "." + key); + // if we care, move this envvar substition stuff to where the file is loaded + string ret = contents.get(section + "." + key); + boost::regex re("\\$\\{(.+)\\}"); + + ret = boost::regex_replace(ret, re, use_envvar); + return ret; } } diff --git a/src/LocalStorage.cpp b/src/LocalStorage.cpp new file mode 100644 index 000000000..f410fa01b --- /dev/null +++ b/src/LocalStorage.cpp @@ -0,0 +1,78 @@ + + +#include +#include +#include +#include "LocalStorage.h" +#include "Config.h" + +using namespace std; +using namespace boost::filesystem; + +namespace storagemanager +{ + +LocalStorage::LocalStorage() +{ + prefix = Config::get()->getValue("LocalStorage", "path"); + cout << "LS: got prefix " << prefix << endl; + if (!is_directory(prefix)) + { + try + { + create_directories(prefix); + } + catch (exception &e) + { + syslog(LOG_CRIT, "Failed to create %s, got: %s", prefix.string().c_str(), e.what()); + throw e; + } + } +} + +LocalStorage::~LocalStorage() +{ +} + +int LocalStorage::copy(const path &source, const path &dest) +{ + boost::system::error_code err; + copy_file(source, dest, copy_option::fail_if_exists, err); + if (!err) + { + errno = err.value(); + return -1; + } + return 0; +} + +path operator+(const path &p1, const path &p2) +{ + path ret(p1); + ret+=p2; + return ret; +} + +int LocalStorage::getObject(const string &source, const string &dest) +{ + return copy(prefix + source, dest); +} + +int LocalStorage::putObject(const string &source, const string &dest) +{ + return copy(source, prefix + dest); +} + +int LocalStorage::copyObject(const string &source, const string &dest) +{ + return copy(prefix + source, prefix + dest); +} + +void LocalStorage::deleteObject(const string &key) +{ + boost::system::error_code err; + + boost::filesystem::remove(prefix + key, err); +} + +} diff --git a/src/LocalStorage.h b/src/LocalStorage.h new file mode 100644 index 000000000..21ea5fc08 --- /dev/null +++ b/src/LocalStorage.h @@ -0,0 +1,30 @@ +#ifndef LOCALSTORAGE_H_ +#define LOCALSTORAGE_H_ + +#include +#include "CloudStorage.h" +#include + +namespace storagemanager +{ + +class LocalStorage : public CloudStorage +{ + public: + LocalStorage(); + virtual ~LocalStorage(); + + int getObject(const std::string &sourceKey, const std::string &destFile); + int putObject(const std::string &sourceFile, const std::string &destKey); + void deleteObject(const std::string &key); + int copyObject(const std::string &sourceKey, const std::string &destKey); + + private: + boost::filesystem::path prefix; + + int copy(const boost::filesystem::path &sourceKey, const boost::filesystem::path &destKey); +}; + +} + +#endif diff --git a/src/S3Storage.cpp b/src/S3Storage.cpp index e3c001243..1a5b74a2b 100644 --- a/src/S3Storage.cpp +++ b/src/S3Storage.cpp @@ -14,7 +14,7 @@ S3Storage::~S3Storage() { } -int S3Storage::getObject(const string &courceKey, const string &destFile) +int S3Storage::getObject(const string &sourceKey, const string &destFile) { return 0; } @@ -28,4 +28,9 @@ void S3Storage::deleteObject(const string &key) { } +int S3Storage::copyObject(const string &sourceKey, const string &destKey) +{ + return 0; +} + } diff --git a/src/S3Storage.h b/src/S3Storage.h index 48066d5c7..04eb2ff59 100644 --- a/src/S3Storage.h +++ b/src/S3Storage.h @@ -17,6 +17,7 @@ class S3Storage : public CloudStorage int getObject(const std::string &sourceKey, const std::string &destFile); int putObject(const std::string &sourceFile, const std::string &destKey); void deleteObject(const std::string &key); + int copyObject(const std::string &sourceKey, const std::string &destKey); private: diff --git a/storagemanager.cnf b/storagemanager.cnf index fa2f4e6a3..50059313b 100644 --- a/storagemanager.cnf +++ b/storagemanager.cnf @@ -11,3 +11,6 @@ region = us-west-1 # ex, by setting AWS_ACCESS_KEY_ID & AWS_SECRET_ACCESS_KEY envvars. # see https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html +[LocalStorage] +path = ${HOME}/fake-cloud/ +