database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-30 19:23:07 +03:00
Code Activity

clang format apply

Browse Source
This commit is contained in:
Leonid Fedorov
2022-01-21 16:43:49 +00:00
parent 6b6411229f
commit 04752ec546
1376 changed files with 393460 additions and 412662 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
storage-manager/src/AppendTask.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "AppendTask.h"
#include "messageFormat.h"
#include "SMLogging.h"
@ -25,7 +24,6 @@ using namespace std;
namespace storagemanager
{
AppendTask::AppendTask(int sock, uint len) : PosixTask(sock, len)
{
}
@ -35,86 +33,86 @@ AppendTask::~AppendTask()
}
#define check_error(msg, ret) \
if (success<0) \
{ \
handleError(msg, errno); \
return ret; \
}
if (success < 0) \
{ \
handleError(msg, errno); \
return ret; \
}
#define min(x, y) (x < y ? x : y)
bool AppendTask::run()
{
SMLogging* logger = SMLogging::get();
int success;
uint8_t cmdbuf[1024] = {0};
ssize_t err;
success = read(cmdbuf, sizeof(append_cmd));
check_error("AppendTask read", false);
append_cmd *cmd = (append_cmd *) cmdbuf;
if (cmd->flen > 1023 - sizeof(*cmd))
{
handleError("AppendTask", ENAMETOOLONG);
return true;
}
success = read(&cmdbuf[sizeof(*cmd)], cmd->flen);
check_error("AppendTask read", false);
#ifdef SM_TRACE
logger->log(LOG_DEBUG,"append %d bytes to %s.",cmd->count,cmd->filename);
#endif
ssize_t readCount = 0, writeCount = 0;
vector<uint8_t> databuf;
uint bufsize = min(100 << 20, cmd->count); // 100 MB
//uint bufsize = cmd->count;
databuf.resize(bufsize);
while (readCount < cmd->count)
{
uint toRead = min(static_cast<uint>(cmd->count - readCount), bufsize);
success = read(&databuf[0], toRead);
check_error("AppendTask read data", false);
if (success==0)
break;
readCount += success;
uint writePos = 0;
SMLogging* logger = SMLogging::get();
int success;
uint8_t cmdbuf[1024] = {0};
ssize_t err;
while (writeCount < readCount)
{
try
{
err = ioc->append(cmd->filename, &databuf[writePos], success - writePos);
}
catch (exception &e)
{
logger->log(LOG_ERR, "AppendTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err <= 0)
break;
writeCount += err;
writePos += err;
}
if (readCount != writeCount)
break;
}
uint8_t respbuf[sizeof(sm_response) + 4];
sm_response *resp = (sm_response *) respbuf;
uint payloadLen = 0;
if (cmd->count != 0 && writeCount == 0)
success = read(cmdbuf, sizeof(append_cmd));
check_error("AppendTask read", false);
append_cmd* cmd = (append_cmd*)cmdbuf;
if (cmd->flen > 1023 - sizeof(*cmd))
{
handleError("AppendTask", ENAMETOOLONG);
return true;
}
success = read(&cmdbuf[sizeof(*cmd)], cmd->flen);
check_error("AppendTask read", false);
#ifdef SM_TRACE
logger->log(LOG_DEBUG, "append %d bytes to %s.", cmd->count, cmd->filename);
#endif
ssize_t readCount = 0, writeCount = 0;
vector<uint8_t> databuf;
uint bufsize = min(100 << 20, cmd->count); // 100 MB
// uint bufsize = cmd->count;
databuf.resize(bufsize);
while (readCount < cmd->count)
{
uint toRead = min(static_cast<uint>(cmd->count - readCount), bufsize);
success = read(&databuf[0], toRead);
check_error("AppendTask read data", false);
if (success == 0)
break;
readCount += success;
uint writePos = 0;
while (writeCount < readCount)
{
payloadLen = 4;
resp->returnCode = -1;
*((int *) &resp[1]) = errno;
try
{
err = ioc->append(cmd->filename, &databuf[writePos], success - writePos);
}
catch (exception& e)
{
logger->log(LOG_ERR, "AppendTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err <= 0)
break;
writeCount += err;
writePos += err;
}
else
resp->returnCode = writeCount;
return write(*resp, payloadLen);
if (readCount != writeCount)
break;
}
uint8_t respbuf[sizeof(sm_response) + 4];
sm_response* resp = (sm_response*)respbuf;
uint payloadLen = 0;
if (cmd->count != 0 && writeCount == 0)
{
payloadLen = 4;
resp->returnCode = -1;
*((int*)&resp[1]) = errno;
}
else
resp->returnCode = writeCount;
return write(*resp, payloadLen);
}
}
} // namespace storagemanager

21
storage-manager/src/AppendTask.h
View File

@ -15,25 +15,22 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "PosixTask.h"
namespace storagemanager
{
class AppendTask : public PosixTask
{
public:
AppendTask(int sock, uint length);
virtual ~AppendTask();
bool run();
private:
AppendTask();
public:
AppendTask(int sock, uint length);
virtual ~AppendTask();
bool run();
private:
AppendTask();
};
}
} // namespace storagemanager

421
storage-manager/src/Cache.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "Cache.h"
#include "Config.h"
#include "Downloader.h"
@ -33,330 +32,328 @@ namespace bf = boost::filesystem;
namespace
{
boost::mutex m;
storagemanager::Cache *inst = NULL;
}
boost::mutex m;
storagemanager::Cache* inst = NULL;
} // namespace
namespace storagemanager
{
Cache * Cache::get()
Cache* Cache::get()
{
if (inst)
return inst;
boost::unique_lock<boost::mutex> s(m);
if (inst)
return inst;
inst = new Cache();
if (inst)
return inst;
boost::unique_lock<boost::mutex> s(m);
if (inst)
return inst;
inst = new Cache();
return inst;
}
Cache::Cache()
{
Config *conf = Config::get();
logger = SMLogging::get();
configListener();
conf->addConfigListener(this);
//cout << "Cache got cache size " << maxCacheSize << endl;
string stmp = conf->getValue("ObjectStorage", "object_size");
if (stmp.empty())
{
logger->log(LOG_CRIT, "ObjectStorage/object_size is not set");
throw runtime_error("Please set ObjectStorage/object_size in the storagemanager.cnf file");
}
try
{
objectSize = stoul(stmp);
}
catch (invalid_argument &)
{
logger->log(LOG_CRIT, "ObjectStorage/object_size is not a number");
throw runtime_error("Please set ObjectStorage/object_size to a number");
}
cachePrefix = conf->getValue("Cache", "path");
if (cachePrefix.empty())
{
logger->log(LOG_CRIT, "Cache/path is not set");
throw runtime_error("Please set Cache/path in the storagemanager.cnf file");
}
try
{
bf::create_directories(cachePrefix);
}
catch (exception &e)
{
logger->log(LOG_CRIT, "Failed to create %s, got: %s", cachePrefix.string().c_str(), e.what());
throw e;
}
//cout << "Cache got cachePrefix " << cachePrefix << endl;
downloader.reset(new Downloader());
stmp = conf->getValue("ObjectStorage", "journal_path");
if (stmp.empty())
{
logger->log(LOG_CRIT, "ObjectStorage/journal_path is not set");
throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file");
}
journalPrefix = stmp;
Config* conf = Config::get();
logger = SMLogging::get();
configListener();
conf->addConfigListener(this);
// cout << "Cache got cache size " << maxCacheSize << endl;
string stmp = conf->getValue("ObjectStorage", "object_size");
if (stmp.empty())
{
logger->log(LOG_CRIT, "ObjectStorage/object_size is not set");
throw runtime_error("Please set ObjectStorage/object_size in the storagemanager.cnf file");
}
try
{
objectSize = stoul(stmp);
}
catch (invalid_argument&)
{
logger->log(LOG_CRIT, "ObjectStorage/object_size is not a number");
throw runtime_error("Please set ObjectStorage/object_size to a number");
}
cachePrefix = conf->getValue("Cache", "path");
if (cachePrefix.empty())
{
logger->log(LOG_CRIT, "Cache/path is not set");
throw runtime_error("Please set Cache/path in the storagemanager.cnf file");
}
try
{
bf::create_directories(cachePrefix);
}
catch (exception& e)
{
logger->log(LOG_CRIT, "Failed to create %s, got: %s", cachePrefix.string().c_str(), e.what());
throw e;
}
// cout << "Cache got cachePrefix " << cachePrefix << endl;
downloader.reset(new Downloader());
stmp = conf->getValue("ObjectStorage", "journal_path");
if (stmp.empty())
{
logger->log(LOG_CRIT, "ObjectStorage/journal_path is not set");
throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file");
}
journalPrefix = stmp;
bf::create_directories(journalPrefix);
try
{
bf::create_directories(journalPrefix);
try
{
bf::create_directories(journalPrefix);
}
catch (exception &e)
{
logger->log(LOG_CRIT, "Failed to create %s, got: %s", journalPrefix.string().c_str(), e.what());
throw e;
}
}
catch (exception& e)
{
logger->log(LOG_CRIT, "Failed to create %s, got: %s", journalPrefix.string().c_str(), e.what());
throw e;
}
}
Cache::~Cache()
{
Config::get()->removeConfigListener(this);
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
delete it->second;
Config::get()->removeConfigListener(this);
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
delete it->second;
}
// be careful using this! SM should be idle. No ongoing reads or writes.
void Cache::validateCacheSize()
{
boost::unique_lock<boost::mutex> s(lru_mutex);
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
it->second->validateCacheSize();
boost::unique_lock<boost::mutex> s(lru_mutex);
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
it->second->validateCacheSize();
}
// new simplified version
void Cache::read(const bf::path &prefix, const vector<string> &keys)
void Cache::read(const bf::path& prefix, const vector<string>& keys)
{
getPCache(prefix).read(keys);
getPCache(prefix).read(keys);
}
void Cache::doneReading(const bf::path &prefix, const vector<string> &keys)
void Cache::doneReading(const bf::path& prefix, const vector<string>& keys)
{
getPCache(prefix).doneReading(keys);
getPCache(prefix).doneReading(keys);
}
void Cache::doneWriting(const bf::path &prefix)
void Cache::doneWriting(const bf::path& prefix)
{
getPCache(prefix).makeSpace(0);
getPCache(prefix).makeSpace(0);
}
const bf::path & Cache::getCachePath() const
const bf::path& Cache::getCachePath() const
{
return cachePrefix;
return cachePrefix;
}
const bf::path & Cache::getJournalPath() const
const bf::path& Cache::getJournalPath() const
{
return journalPrefix;
return journalPrefix;
}
const bf::path Cache::getCachePath(const bf::path &prefix) const
const bf::path Cache::getCachePath(const bf::path& prefix) const
{
return cachePrefix/prefix;
return cachePrefix / prefix;
}
const bf::path Cache::getJournalPath(const bf::path &prefix) const
const bf::path Cache::getJournalPath(const bf::path& prefix) const
{
return journalPrefix/prefix;
}
void Cache::exists(const bf::path &prefix, const vector<string> &keys, vector<bool> *out)
{
getPCache(prefix).exists(keys, out);
return journalPrefix / prefix;
}
bool Cache::exists(const bf::path &prefix, const string &key)
void Cache::exists(const bf::path& prefix, const vector<string>& keys, vector<bool>* out)
{
return getPCache(prefix).exists(key);
getPCache(prefix).exists(keys, out);
}
void Cache::newObject(const bf::path &prefix, const string &key, size_t size)
bool Cache::exists(const bf::path& prefix, const string& key)
{
getPCache(prefix).newObject(key, size);
return getPCache(prefix).exists(key);
}
void Cache::newJournalEntry(const bf::path &prefix, size_t size)
void Cache::newObject(const bf::path& prefix, const string& key, size_t size)
{
getPCache(prefix).newJournalEntry(size);
getPCache(prefix).newObject(key, size);
}
void Cache::deletedJournal(const bf::path &prefix, size_t size)
void Cache::newJournalEntry(const bf::path& prefix, size_t size)
{
getPCache(prefix).deletedJournal(size);
getPCache(prefix).newJournalEntry(size);
}
void Cache::deletedObject(const bf::path &prefix, const string &key, size_t size)
void Cache::deletedJournal(const bf::path& prefix, size_t size)
{
getPCache(prefix).deletedObject(key, size);
getPCache(prefix).deletedJournal(size);
}
void Cache::deletedObject(const bf::path& prefix, const string& key, size_t size)
{
getPCache(prefix).deletedObject(key, size);
}
void Cache::setMaxCacheSize(size_t size)
{
boost::unique_lock<boost::mutex> s(lru_mutex);
maxCacheSize = size;
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
it->second->setMaxCacheSize(size);
boost::unique_lock<boost::mutex> s(lru_mutex);
maxCacheSize = size;
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
it->second->setMaxCacheSize(size);
}
void Cache::makeSpace(const bf::path &prefix, size_t size)
void Cache::makeSpace(const bf::path& prefix, size_t size)
{
getPCache(prefix).makeSpace(size);
getPCache(prefix).makeSpace(size);
}
size_t Cache::getMaxCacheSize() const
{
return maxCacheSize;
return maxCacheSize;
}
void Cache::rename(const bf::path &prefix, const string &oldKey, const string &newKey, ssize_t sizediff)
void Cache::rename(const bf::path& prefix, const string& oldKey, const string& newKey, ssize_t sizediff)
{
getPCache(prefix).rename(oldKey, newKey, sizediff);
getPCache(prefix).rename(oldKey, newKey, sizediff);
}
int Cache::ifExistsThenDelete(const bf::path &prefix, const string &key)
int Cache::ifExistsThenDelete(const bf::path& prefix, const string& key)
{
return getPCache(prefix).ifExistsThenDelete(key);
return getPCache(prefix).ifExistsThenDelete(key);
}
void Cache::printKPIs() const
{
downloader->printKPIs();
downloader->printKPIs();
}
size_t Cache::getCurrentCacheSize()
{
size_t totalSize = 0;
size_t totalSize = 0;
boost::unique_lock<boost::mutex> s(lru_mutex);
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
totalSize += it->second->getCurrentCacheSize();
return totalSize;
boost::unique_lock<boost::mutex> s(lru_mutex);
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
totalSize += it->second->getCurrentCacheSize();
return totalSize;
}
size_t Cache::getCurrentCacheElementCount()
{
size_t totalCount = 0;
size_t totalCount = 0;
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
totalCount += it->second->getCurrentCacheElementCount();
return totalCount;
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
totalCount += it->second->getCurrentCacheElementCount();
return totalCount;
}
size_t Cache::getCurrentCacheSize(const bf::path &prefix)
size_t Cache::getCurrentCacheSize(const bf::path& prefix)
{
return getPCache(prefix).getCurrentCacheSize();
return getPCache(prefix).getCurrentCacheSize();
}
size_t Cache::getCurrentCacheElementCount(const bf::path &prefix)
size_t Cache::getCurrentCacheElementCount(const bf::path& prefix)
{
return getPCache(prefix).getCurrentCacheElementCount();
return getPCache(prefix).getCurrentCacheElementCount();
}
void Cache::reset()
{
boost::unique_lock<boost::mutex> s(lru_mutex);
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
it->second->reset();
boost::unique_lock<boost::mutex> s(lru_mutex);
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
it->second->reset();
}
void Cache::newPrefix(const bf::path &prefix)
void Cache::newPrefix(const bf::path& prefix)
{
boost::unique_lock<boost::mutex> s(lru_mutex);
//cerr << "Cache: making new prefix " << prefix.string() << endl;
assert(prefixCaches.find(prefix) == prefixCaches.end());
prefixCaches[prefix] = NULL;
boost::unique_lock<boost::mutex> s(lru_mutex);
// cerr << "Cache: making new prefix " << prefix.string() << endl;
assert(prefixCaches.find(prefix) == prefixCaches.end());
prefixCaches[prefix] = NULL;
s.unlock();
PrefixCache* pCache = new PrefixCache(prefix);
s.lock();
prefixCaches[prefix] = pCache;
}
void Cache::dropPrefix(const bf::path& prefix)
{
boost::unique_lock<boost::mutex> s(lru_mutex);
auto* pCache = prefixCaches[prefix];
prefixCaches.erase(prefix);
s.unlock();
delete pCache;
}
inline PrefixCache& Cache::getPCache(const bf::path& prefix)
{
boost::unique_lock<boost::mutex> s(lru_mutex);
// cerr << "Getting pcache for " << prefix.string() << endl;
PrefixCache* ret;
auto it = prefixCaches.find(prefix);
assert(it != prefixCaches.end());
ret = it->second;
while (ret == NULL) // wait for the new PC to init
{
s.unlock();
PrefixCache *pCache = new PrefixCache(prefix);
sleep(1);
s.lock();
prefixCaches[prefix] = pCache;
ret = prefixCaches[prefix];
}
return *ret;
}
void Cache::dropPrefix(const bf::path &prefix)
Downloader* Cache::getDownloader() const
{
boost::unique_lock<boost::mutex> s(lru_mutex);
auto *pCache = prefixCaches[prefix];
prefixCaches.erase(prefix);
s.unlock();
delete pCache;
}
inline PrefixCache & Cache::getPCache(const bf::path &prefix)
{
boost::unique_lock<boost::mutex> s(lru_mutex);
//cerr << "Getting pcache for " << prefix.string() << endl;
PrefixCache *ret;
auto it = prefixCaches.find(prefix);
assert(it != prefixCaches.end());
ret = it->second;
while (ret == NULL) // wait for the new PC to init
{
s.unlock();
sleep(1);
s.lock();
ret = prefixCaches[prefix];
}
return *ret;
}
Downloader * Cache::getDownloader() const
{
return downloader.get();
return downloader.get();
}
void Cache::shutdown()
{
boost::unique_lock<boost::mutex> s(lru_mutex);
downloader.reset();
boost::unique_lock<boost::mutex> s(lru_mutex);
downloader.reset();
}
void Cache::configListener()
{
Config *conf = Config::get();
SMLogging* logger = SMLogging::get();
Config* conf = Config::get();
SMLogging* logger = SMLogging::get();
if (maxCacheSize == 0)
maxCacheSize = 2147483648;
string stmp = conf->getValue("Cache", "cache_size");
if (stmp.empty())
if (maxCacheSize == 0)
maxCacheSize = 2147483648;
string stmp = conf->getValue("Cache", "cache_size");
if (stmp.empty())
{
logger->log(LOG_CRIT, "Cache/cache_size is not set. Using current value = %zi", maxCacheSize);
}
try
{
size_t newMaxCacheSize = stoull(stmp);
if (newMaxCacheSize != maxCacheSize)
{
logger->log(LOG_CRIT, "Cache/cache_size is not set. Using current value = %zi",maxCacheSize);
}
try
{
size_t newMaxCacheSize = stoull(stmp);
if (newMaxCacheSize != maxCacheSize)
{
if (newMaxCacheSize >= MIN_CACHE_SIZE)
{
setMaxCacheSize(newMaxCacheSize);
logger->log(LOG_INFO, "Cache/cache_size = %zi",maxCacheSize);
}
else
{
logger->log(LOG_CRIT, "Cache/cache_size is below %u. Check value and suffix are correct in configuration file. Using current value = %zi",MIN_CACHE_SIZE,maxCacheSize);
}
}
}
catch (invalid_argument &)
{
logger->log(LOG_CRIT, "Cache/cache_size is not a number. Using current value = %zi",maxCacheSize);
if (newMaxCacheSize >= MIN_CACHE_SIZE)
{
setMaxCacheSize(newMaxCacheSize);
logger->log(LOG_INFO, "Cache/cache_size = %zi", maxCacheSize);
}
else
{
logger->log(LOG_CRIT,
"Cache/cache_size is below %u. Check value and suffix are correct in configuration file. "
"Using current value = %zi",
MIN_CACHE_SIZE, maxCacheSize);
}
}
}
catch (invalid_argument&)
{
logger->log(LOG_CRIT, "Cache/cache_size is not a number. Using current value = %zi", maxCacheSize);
}
}
}
} // namespace storagemanager

148
storage-manager/src/Cache.h
View File

@ -17,12 +17,11 @@
#pragma once
/* PrefixCache manages the cache for one prefix managed by SM.
/* PrefixCache manages the cache for one prefix managed by SM.
Cache is a map of prefix -> Cache, and holds the items
that should be centralized like the Downloader
*/
#include "Downloader.h"
#include "SMLogging.h"
#include "PrefixCache.h"
@ -43,81 +42,78 @@
namespace storagemanager
{
class Cache : public boost::noncopyable , public ConfigListener
class Cache : public boost::noncopyable, public ConfigListener
{
public:
static Cache *get();
virtual ~Cache();
//reading fcns
// read() marks objects to be read s.t. they do not get flushed.
// after reading them, unlock the 'logical file', and call doneReading().
void read(const boost::filesystem::path &prefix, const std::vector<std::string> &keys);
void doneReading(const boost::filesystem::path &prefix, const std::vector<std::string> &keys);
bool exists(const boost::filesystem::path &prefix, const std::string &key);
void exists(const boost::filesystem::path &prefix, const std::vector<std::string> &keys,
std::vector<bool> *out);
// writing fcns
// new*() fcns tell the Cache data was added. After writing a set of objects,
// unlock the 'logical file', and call doneWriting().
void newObject(const boost::filesystem::path &prefix, const std::string &key, size_t size);
void newJournalEntry(const boost::filesystem::path &prefix, size_t size);
void doneWriting(const boost::filesystem::path &prefix);
void deletedObject(const boost::filesystem::path &prefix, const std::string &key, size_t size);
void deletedJournal(const boost::filesystem::path &prefix, size_t size);
// an 'atomic' existence check & delete. Covers the object and journal. Does not delete the files.
// returns 0 if it didn't exist, 1 if the object exists, 2 if the journal exists, and 3 (1 | 2) if both exist
// This should be called while holding the file lock for key because it touches the journal file.
int ifExistsThenDelete(const boost::filesystem::path &prefix, const std::string &key);
// rename is used when an old obj gets merged with its journal file
// the size will change in that process; sizediff is by how much
void rename(const boost::filesystem::path &prefix, const std::string &oldKey,
const std::string &newKey, ssize_t sizediff);
void setMaxCacheSize(size_t size);
void makeSpace(const boost::filesystem::path &prefix, size_t size);
size_t getCurrentCacheSize();
size_t getCurrentCacheElementCount();
size_t getCurrentCacheSize(const boost::filesystem::path &prefix);
size_t getCurrentCacheElementCount(const boost::filesystem::path &prefix);
size_t getMaxCacheSize() const;
Downloader * getDownloader() const;
void newPrefix(const boost::filesystem::path &prefix);
void dropPrefix(const boost::filesystem::path &prefix);
void shutdown();
void printKPIs() const;
// test helpers
const boost::filesystem::path &getCachePath() const;
const boost::filesystem::path &getJournalPath() const;
const boost::filesystem::path getCachePath(const boost::filesystem::path &prefix) const;
const boost::filesystem::path getJournalPath(const boost::filesystem::path &prefix) const;
// this will delete everything in the Cache and journal paths, and empty all Cache structures.
void reset();
void validateCacheSize();
virtual void configListener() override;
private:
Cache();
SMLogging *logger;
boost::filesystem::path cachePrefix;
boost::filesystem::path journalPrefix;
size_t maxCacheSize;
size_t objectSize;
boost::scoped_ptr<Downloader> downloader;
PrefixCache & getPCache(const boost::filesystem::path &prefix);
public:
static Cache* get();
virtual ~Cache();
std::map<boost::filesystem::path, PrefixCache *> prefixCaches;
mutable boost::mutex lru_mutex; // protects the prefixCaches
// reading fcns
// read() marks objects to be read s.t. they do not get flushed.
// after reading them, unlock the 'logical file', and call doneReading().
void read(const boost::filesystem::path& prefix, const std::vector<std::string>& keys);
void doneReading(const boost::filesystem::path& prefix, const std::vector<std::string>& keys);
bool exists(const boost::filesystem::path& prefix, const std::string& key);
void exists(const boost::filesystem::path& prefix, const std::vector<std::string>& keys,
std::vector<bool>* out);
// writing fcns
// new*() fcns tell the Cache data was added. After writing a set of objects,
// unlock the 'logical file', and call doneWriting().
void newObject(const boost::filesystem::path& prefix, const std::string& key, size_t size);
void newJournalEntry(const boost::filesystem::path& prefix, size_t size);
void doneWriting(const boost::filesystem::path& prefix);
void deletedObject(const boost::filesystem::path& prefix, const std::string& key, size_t size);
void deletedJournal(const boost::filesystem::path& prefix, size_t size);
// an 'atomic' existence check & delete. Covers the object and journal. Does not delete the files.
// returns 0 if it didn't exist, 1 if the object exists, 2 if the journal exists, and 3 (1 | 2) if both
// exist This should be called while holding the file lock for key because it touches the journal file.
int ifExistsThenDelete(const boost::filesystem::path& prefix, const std::string& key);
// rename is used when an old obj gets merged with its journal file
// the size will change in that process; sizediff is by how much
void rename(const boost::filesystem::path& prefix, const std::string& oldKey, const std::string& newKey,
ssize_t sizediff);
void setMaxCacheSize(size_t size);
void makeSpace(const boost::filesystem::path& prefix, size_t size);
size_t getCurrentCacheSize();
size_t getCurrentCacheElementCount();
size_t getCurrentCacheSize(const boost::filesystem::path& prefix);
size_t getCurrentCacheElementCount(const boost::filesystem::path& prefix);
size_t getMaxCacheSize() const;
Downloader* getDownloader() const;
void newPrefix(const boost::filesystem::path& prefix);
void dropPrefix(const boost::filesystem::path& prefix);
void shutdown();
void printKPIs() const;
// test helpers
const boost::filesystem::path& getCachePath() const;
const boost::filesystem::path& getJournalPath() const;
const boost::filesystem::path getCachePath(const boost::filesystem::path& prefix) const;
const boost::filesystem::path getJournalPath(const boost::filesystem::path& prefix) const;
// this will delete everything in the Cache and journal paths, and empty all Cache structures.
void reset();
void validateCacheSize();
virtual void configListener() override;
private:
Cache();
SMLogging* logger;
boost::filesystem::path cachePrefix;
boost::filesystem::path journalPrefix;
size_t maxCacheSize;
size_t objectSize;
boost::scoped_ptr<Downloader> downloader;
PrefixCache& getPCache(const boost::filesystem::path& prefix);
std::map<boost::filesystem::path, PrefixCache*> prefixCaches;
mutable boost::mutex lru_mutex; // protects the prefixCaches
};
}
} // namespace storagemanager

30
storage-manager/src/ClientRequestProcessor.cpp
View File

@ -15,20 +15,18 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "ClientRequestProcessor.h"
#include "ProcessTask.h"
#include <sys/types.h>
namespace
{
storagemanager::ClientRequestProcessor *crp = NULL;
storagemanager::ClientRequestProcessor* crp = NULL;
boost::mutex m;
};
}; // namespace
namespace storagemanager
{
ClientRequestProcessor::ClientRequestProcessor()
{
}
@ -37,28 +35,26 @@ ClientRequestProcessor::~ClientRequestProcessor()
{
}
ClientRequestProcessor * ClientRequestProcessor::get()
ClientRequestProcessor* ClientRequestProcessor::get()
{
if (crp)
return crp;
boost::mutex::scoped_lock s(m);
if (crp)
return crp;
crp = new ClientRequestProcessor();
if (crp)
return crp;
boost::mutex::scoped_lock s(m);
if (crp)
return crp;
crp = new ClientRequestProcessor();
return crp;
}
void ClientRequestProcessor::processRequest(int sock, uint len)
{
boost::shared_ptr<ThreadPool::Job> t(new ProcessTask(sock, len));
threadPool.addJob(t);
boost::shared_ptr<ThreadPool::Job> t(new ProcessTask(sock, len));
threadPool.addJob(t);
}
void ClientRequestProcessor::shutdown()
{
delete crp;
delete crp;
}
}
} // namespace storagemanager

22
storage-manager/src/ClientRequestProcessor.h
View File

@ -15,29 +15,25 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "ThreadPool.h"
#include <sys/types.h>
namespace storagemanager
{
class ClientRequestProcessor : public boost::noncopyable
{
public:
static ClientRequestProcessor *get();
virtual ~ClientRequestProcessor();
public:
static ClientRequestProcessor* get();
virtual ~ClientRequestProcessor();
void processRequest(int sock, uint len);
void shutdown();
void processRequest(int sock, uint len);
void shutdown();
private:
ClientRequestProcessor();
ThreadPool threadPool;
private:
ClientRequestProcessor();
ThreadPool threadPool;
};
}
} // namespace storagemanager

88
storage-manager/src/CloudStorage.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "CloudStorage.h"
#include "Config.h"
#include "S3Storage.h"
@ -30,66 +29,63 @@ using namespace std;
namespace
{
boost::mutex m;
storagemanager::CloudStorage *inst = NULL;
string tolower(const string &s)
boost::mutex m;
storagemanager::CloudStorage* inst = NULL;
string tolower(const string& s)
{
string ret(s);
for (uint i = 0; i < ret.length(); i++)
ret[i] = ::tolower(ret[i]);
return ret;
string ret(s);
for (uint i = 0; i < ret.length(); i++)
ret[i] = ::tolower(ret[i]);
return ret;
}
}
} // namespace
namespace storagemanager
{
CloudStorage * CloudStorage::get()
CloudStorage* CloudStorage::get()
{
if (inst)
return inst;
SMLogging* logger = SMLogging::get();
Config *conf = Config::get();
string type = tolower(conf->getValue("ObjectStorage", "service"));
boost::mutex::scoped_lock s(m);
if (inst)
return inst;
if (type == "s3")
inst = new S3Storage();
else if (type == "local" || type == "localstorage")
inst = new LocalStorage();
else {
logger->log(LOG_CRIT,"CloudStorage: got unknown service provider: %s", type.c_str());
throw runtime_error("CloudStorage: got unknown service provider");
}
if (inst)
return inst;
SMLogging* logger = SMLogging::get();
Config* conf = Config::get();
string type = tolower(conf->getValue("ObjectStorage", "service"));
boost::mutex::scoped_lock s(m);
if (inst)
return inst;
if (type == "s3")
inst = new S3Storage();
else if (type == "local" || type == "localstorage")
inst = new LocalStorage();
else
{
logger->log(LOG_CRIT, "CloudStorage: got unknown service provider: %s", type.c_str());
throw runtime_error("CloudStorage: got unknown service provider");
}
return inst;
}
CloudStorage::CloudStorage()
{
logger = SMLogging::get();
bytesUploaded = bytesDownloaded = objectsDeleted = objectsCopied = objectsGotten = objectsPut =
existenceChecks = 0;
logger = SMLogging::get();
bytesUploaded = bytesDownloaded = objectsDeleted = objectsCopied = objectsGotten = objectsPut =
existenceChecks = 0;
}
void CloudStorage::printKPIs() const
{
cout << "CloudStorage" << endl;
cout << "\tbytesUplaoded = " << bytesUploaded << endl;
cout << "\tbytesDownloaded = " << bytesDownloaded << endl;
cout << "\tobjectsDeleted = " << objectsDeleted << endl;
cout << "\tobjectsCopied = " << objectsCopied << endl;
cout << "\tobjectsGotten = " << objectsGotten << endl;
cout << "\tobjectsPut = " << objectsPut << endl;
cout << "\texistenceChecks = " << existenceChecks << endl;
cout << "CloudStorage" << endl;
cout << "\tbytesUplaoded = " << bytesUploaded << endl;
cout << "\tbytesDownloaded = " << bytesDownloaded << endl;
cout << "\tobjectsDeleted = " << objectsDeleted << endl;
cout << "\tobjectsCopied = " << objectsCopied << endl;
cout << "\tobjectsGotten = " << objectsGotten << endl;
cout << "\tobjectsPut = " << objectsPut << endl;
cout << "\texistenceChecks = " << existenceChecks << endl;
}
}
} // namespace storagemanager

57
storage-manager/src/CloudStorage.h
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <string>
@ -24,36 +23,34 @@
namespace storagemanager
{
class CloudStorage
{
public:
virtual ~CloudStorage(){};
/* These behave like syscalls. return code -1 means an error, and errno is set */
virtual int getObject(const std::string &sourceKey, const std::string &destFile, size_t *size = NULL) = 0;
virtual int getObject(const std::string &sourceKey, boost::shared_array<uint8_t> *data, size_t *size = NULL) = 0;
virtual int putObject(const std::string &sourceFile, const std::string &destKey) = 0;
virtual int putObject(const boost::shared_array<uint8_t> data, size_t len, const std::string &destKey) = 0;
virtual int deleteObject(const std::string &key) = 0;
virtual int copyObject(const std::string &sourceKey, const std::string &destKey) = 0;
virtual int exists(const std::string &key, bool *out) = 0;
virtual void printKPIs() const;
// this will return a CloudStorage instance of the type specified in StorageManager.cnf
static CloudStorage *get();
protected:
SMLogging *logger;
CloudStorage();
// some KPIs
size_t bytesUploaded, bytesDownloaded, objectsDeleted, objectsCopied, objectsGotten,
objectsPut, existenceChecks;
private:
public:
virtual ~CloudStorage(){};
/* These behave like syscalls. return code -1 means an error, and errno is set */
virtual int getObject(const std::string& sourceKey, const std::string& destFile, size_t* size = NULL) = 0;
virtual int getObject(const std::string& sourceKey, boost::shared_array<uint8_t>* data,
size_t* size = NULL) = 0;
virtual int putObject(const std::string& sourceFile, const std::string& destKey) = 0;
virtual int putObject(const boost::shared_array<uint8_t> data, size_t len, const std::string& destKey) = 0;
virtual int deleteObject(const std::string& key) = 0;
virtual int copyObject(const std::string& sourceKey, const std::string& destKey) = 0;
virtual int exists(const std::string& key, bool* out) = 0;
virtual void printKPIs() const;
// this will return a CloudStorage instance of the type specified in StorageManager.cnf
static CloudStorage* get();
protected:
SMLogging* logger;
CloudStorage();
// some KPIs
size_t bytesUploaded, bytesDownloaded, objectsDeleted, objectsCopied, objectsGotten, objectsPut,
existenceChecks;
private:
};
}
} // namespace storagemanager

279
storage-manager/src/Config.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "Config.h"
// This one is the build system config
@ -40,191 +39,193 @@ using namespace std;
namespace
{
boost::mutex m;
storagemanager::Config *inst = NULL;
}
boost::mutex m;
storagemanager::Config* inst = NULL;
} // namespace
namespace storagemanager
{
Config * Config::get()
Config* Config::get()
{
if (inst)
return inst;
boost::mutex::scoped_lock s(m);
if (inst)
return inst;
inst = new Config();
if (inst)
return inst;
boost::mutex::scoped_lock s(m);
if (inst)
return inst;
inst = new Config();
return inst;
}
Config * Config::get(const string &configFile)
Config* Config::get(const string& configFile)
{
if (inst)
return inst;
boost::mutex::scoped_lock s(m);
if (inst)
return inst;
inst = new Config(configFile);
if (inst)
return inst;
boost::mutex::scoped_lock s(m);
if (inst)
return inst;
inst = new Config(configFile);
return inst;
}
Config::Config() : die(false)
{
/* This will search the current directory,
then $COLUMNSTORE_INSTALL_DIR/etc,
then /etc
looking for storagemanager.cnf
We can change this however we need later.
*/
//char *cs_install_dir = getenv("COLUMNSTORE_INSTALL_DIR");
vector<string> paths;
// the paths to search in order
paths.push_back(".");
paths.push_back(string(MCSSYSCONFDIR) + "/columnstore");
paths.push_back("/etc");
for (uint i = 0; i < paths.size(); i++)
/* This will search the current directory,
then $COLUMNSTORE_INSTALL_DIR/etc,
then /etc
looking for storagemanager.cnf
We can change this however we need later.
*/
// char *cs_install_dir = getenv("COLUMNSTORE_INSTALL_DIR");
vector<string> paths;
// the paths to search in order
paths.push_back(".");
paths.push_back(string(MCSSYSCONFDIR) + "/columnstore");
paths.push_back("/etc");
for (uint i = 0; i < paths.size(); i++)
{
// cout << "Config: searching '" << paths[i] << "'" << endl;
if (boost::filesystem::exists(paths[i] + "/storagemanager.cnf"))
{
//cout << "Config: searching '" << paths[i] << "'" << endl;
if (boost::filesystem::exists(paths[i] + "/storagemanager.cnf"))
{
filename = paths[i] + "/storagemanager.cnf";
SMLogging::get()->log(LOG_DEBUG, "Using the config file found at %s", filename.c_str());
break;
}
filename = paths[i] + "/storagemanager.cnf";
SMLogging::get()->log(LOG_DEBUG, "Using the config file found at %s", filename.c_str());
break;
}
if (filename.empty())
throw runtime_error("Config: Could not find the config file for StorageManager");
reloadInterval = boost::posix_time::seconds(60);
last_mtime = {0, 0};
reload();
reloader = boost::thread([this] { this->reloadThreadFcn(); });
}
if (filename.empty())
throw runtime_error("Config: Could not find the config file for StorageManager");
reloadInterval = boost::posix_time::seconds(60);
last_mtime = {0, 0};
reload();
reloader = boost::thread([this] { this->reloadThreadFcn(); });
}
Config::Config(const string &configFile) : filename(configFile), die(false)
Config::Config(const string& configFile) : filename(configFile), die(false)
{
if (!bf::is_regular_file(configFile))
throw runtime_error("Config: Could not find the config file for StorageManager");
reloadInterval = boost::posix_time::seconds(60);
last_mtime = {0, 0};
reload();
reloader = boost::thread([this] { this->reloadThreadFcn(); });
if (!bf::is_regular_file(configFile))
throw runtime_error("Config: Could not find the config file for StorageManager");
reloadInterval = boost::posix_time::seconds(60);
last_mtime = {0, 0};
reload();
reloader = boost::thread([this] { this->reloadThreadFcn(); });
}
Config::~Config()
{
die = true;
reloader.interrupt();
reloader.join();
die = true;
reloader.interrupt();
reloader.join();
}
void Config::reloadThreadFcn()
{
while (!die)
while (!die)
{
try
{
try
{
if (reload())
{
for (auto& listener : configListeners)
listener->configListener();
}
boost::this_thread::sleep(reloadInterval);
}
catch (boost::property_tree::ini_parser_error &e)
{
// log the error. how to log things now?
}
catch (boost::thread_interrupted)
{}
if (reload())
{
for (auto& listener : configListeners)
listener->configListener();
}
boost::this_thread::sleep(reloadInterval);
}
catch (boost::property_tree::ini_parser_error& e)
{
// log the error. how to log things now?
}
catch (boost::thread_interrupted)
{
}
}
}
bool Config::reload()
{
bool rtn = false;
struct stat statbuf;
int err = stat(filename.c_str(), &statbuf);
if (err)
{
SMLogging::get()->log(LOG_ERR, "Config::reload error %s", filename.c_str());
return rtn;
}
if ((statbuf.st_mtim.tv_sec == last_mtime.tv_sec) && (statbuf.st_mtim.tv_nsec == last_mtime.tv_nsec))
return rtn;
last_mtime = statbuf.st_mtim;
rtn = true;
boost::unique_lock<boost::mutex> s(mutex);
contents.clear();
boost::property_tree::ini_parser::read_ini(filename, contents);
bool rtn = false;
struct stat statbuf;
int err = stat(filename.c_str(), &statbuf);
if (err)
{
SMLogging::get()->log(LOG_ERR, "Config::reload error %s", filename.c_str());
return rtn;
}
if ((statbuf.st_mtim.tv_sec == last_mtime.tv_sec) && (statbuf.st_mtim.tv_nsec == last_mtime.tv_nsec))
return rtn;
last_mtime = statbuf.st_mtim;
rtn = true;
boost::unique_lock<boost::mutex> s(mutex);
contents.clear();
boost::property_tree::ini_parser::read_ini(filename, contents);
return rtn;
}
string use_envvar(const boost::smatch &envvar)
string use_envvar(const boost::smatch& envvar)
{
char *env = getenv(envvar[1].str().c_str());
return (env ? env : "");
char* env = getenv(envvar[1].str().c_str());
return (env ? env : "");
}
string expand_numbers(const boost::smatch &match)
string expand_numbers(const boost::smatch& match)
{
long long num = stol(match[1].str());
char suffix = (char) ::tolower(match[2].str()[0]);
long long num = stol(match[1].str());
char suffix = (char)::tolower(match[2].str()[0]);
if (suffix == 't')
num <<= 40;
else if (suffix == 'g')
num <<= 30;
else if (suffix == 'm')
num <<= 20;
else if (suffix == 'k')
num <<= 10;
return ::to_string(num);
if (suffix == 't')
num <<= 40;
else if (suffix == 'g')
num <<= 30;
else if (suffix == 'm')
num <<= 20;
else if (suffix == 'k')
num <<= 10;
return ::to_string(num);
}
string Config::getValue(const string &section, const string &key) const
string Config::getValue(const string& section, const string& key) const
{
// if we care, move this envvar substition stuff to where the file is loaded
string ret;
boost::unique_lock<boost::mutex> s(mutex);
try {
ret = contents.get<string>(section + "." + key);
}
catch (...) {
return ""; // debating whether it's necessary to tell the caller there was no entry.
}
s.unlock();
boost::regex re("\\$\\{(.+)\\}");
ret = boost::regex_replace(ret, re, use_envvar);
// do the numeric substitutions. ex, the suffixes m, k, g
// ehhhhh. going to end up turning a string to a number, to a string, and then to a number again
// don't like that. OTOH who cares.
boost::regex num_re("^([[:digit:]]+)([mMkKgG])$", boost::regex::extended);
ret = boost::regex_replace(ret, num_re, expand_numbers);
return ret;
// if we care, move this envvar substition stuff to where the file is loaded
string ret;
boost::unique_lock<boost::mutex> s(mutex);
try
{
ret = contents.get<string>(section + "." + key);
}
catch (...)
{
return ""; // debating whether it's necessary to tell the caller there was no entry.
}
s.unlock();
boost::regex re("\\$\\{(.+)\\}");
ret = boost::regex_replace(ret, re, use_envvar);
// do the numeric substitutions. ex, the suffixes m, k, g
// ehhhhh. going to end up turning a string to a number, to a string, and then to a number again
// don't like that. OTOH who cares.
boost::regex num_re("^([[:digit:]]+)([mMkKgG])$", boost::regex::extended);
ret = boost::regex_replace(ret, num_re, expand_numbers);
return ret;
}
void Config::addConfigListener(ConfigListener *listener)
void Config::addConfigListener(ConfigListener* listener)
{
configListeners.push_back(listener);
configListeners.push_back(listener);
}
void Config::removeConfigListener(ConfigListener *listener)
void Config::removeConfigListener(ConfigListener* listener)
{
auto iterator = std::find(configListeners.begin(), configListeners.end(), listener);
auto iterator = std::find(configListeners.begin(), configListeners.end(), listener);
if (iterator != configListeners.end())
configListeners.erase(iterator);
if (iterator != configListeners.end())
configListeners.erase(iterator);
}
}
} // namespace storagemanager

63
storage-manager/src/Config.h
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <boost/property_tree/ptree.hpp>
@ -31,44 +30,40 @@ namespace storagemanager
{
class ConfigListener
{
public:
virtual ~ConfigListener(){};
virtual void configListener() = 0;
public:
virtual ~ConfigListener(){};
virtual void configListener() = 0;
};
class Config : public boost::noncopyable
{
public:
static Config *get();
virtual ~Config();
std::string getValue(const std::string &section, const std::string &key) const;
// for testing, lets caller specify a config file to use
static Config *get(const std::string &);
public:
static Config* get();
virtual ~Config();
void addConfigListener(ConfigListener *listener);
void removeConfigListener(ConfigListener *listener);
std::string getValue(const std::string& section, const std::string& key) const;
private:
Config();
Config(const std::string &);
bool reload();
void reloadThreadFcn();
std::vector<ConfigListener *> configListeners;
struct ::timespec last_mtime;
mutable boost::mutex mutex;
boost::thread reloader;
boost::posix_time::time_duration reloadInterval;
std::string filename;
boost::property_tree::ptree contents;
bool die;
// for testing, lets caller specify a config file to use
static Config* get(const std::string&);
void addConfigListener(ConfigListener* listener);
void removeConfigListener(ConfigListener* listener);
private:
Config();
Config(const std::string&);
bool reload();
void reloadThreadFcn();
std::vector<ConfigListener*> configListeners;
struct ::timespec last_mtime;
mutable boost::mutex mutex;
boost::thread reloader;
boost::posix_time::time_duration reloadInterval;
std::string filename;
boost::property_tree::ptree contents;
bool die;
};
}
} // namespace storagemanager

95
storage-manager/src/CopyTask.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "CopyTask.h"
#include <errno.h>
#include "SMLogging.h"
@ -25,7 +24,6 @@ using namespace std;
namespace storagemanager
{
CopyTask::CopyTask(int sock, uint len) : PosixTask(sock, len)
{
}
@ -35,55 +33,56 @@ CopyTask::~CopyTask()
}
#define check_error(msg, ret) \
if (success<0) \
{ \
handleError(msg, errno); \
return ret; \
}
if (success < 0) \
{ \
handleError(msg, errno); \
return ret; \
}
bool CopyTask::run()
{
int success;
SMLogging* logger = SMLogging::get();
int success;
SMLogging* logger = SMLogging::get();
uint8_t buf[2048] = {0};
if (getLength() > 2047)
{
handleError("CopyTask read", ENAMETOOLONG);
return true;
}
success = read(buf, getLength());
check_error("CopyTask read", false);
copy_cmd *cmd = (copy_cmd *) buf;
string filename1(cmd->file1.filename, cmd->file1.flen); // need to copy this in case it's not null terminated
f_name *filename2 = (f_name *) &buf[sizeof(copy_cmd) + cmd->file1.flen];
#ifdef SM_TRACE
logger->log(LOG_DEBUG,"copy %s to %s.",filename1.c_str(),filename2->filename);
#endif
int err;
try
{
err = ioc->copyFile(filename1.c_str(), filename2->filename);
}
catch (exception &e)
{
logger->log(LOG_ERR, "CopyTask: caught %s", e.what());
err = -1;
errno = EIO;
}
if (err)
{
handleError("CopyTask copy", errno);
return true;
}
sm_response *resp = (sm_response *) buf;
resp->returnCode = 0;
return write(*resp, 0);
uint8_t buf[2048] = {0};
if (getLength() > 2047)
{
handleError("CopyTask read", ENAMETOOLONG);
return true;
}
success = read(buf, getLength());
check_error("CopyTask read", false);
copy_cmd* cmd = (copy_cmd*)buf;
string filename1(cmd->file1.filename,
cmd->file1.flen); // need to copy this in case it's not null terminated
f_name* filename2 = (f_name*)&buf[sizeof(copy_cmd) + cmd->file1.flen];
#ifdef SM_TRACE
logger->log(LOG_DEBUG, "copy %s to %s.", filename1.c_str(), filename2->filename);
#endif
int err;
try
{
err = ioc->copyFile(filename1.c_str(), filename2->filename);
}
catch (exception& e)
{
logger->log(LOG_ERR, "CopyTask: caught %s", e.what());
err = -1;
errno = EIO;
}
if (err)
{
handleError("CopyTask copy", errno);
return true;
}
sm_response* resp = (sm_response*)buf;
resp->returnCode = 0;
return write(*resp, 0);
}
}
} // namespace storagemanager

21
storage-manager/src/CopyTask.h
View File

@ -15,25 +15,22 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "PosixTask.h"
namespace storagemanager
{
class CopyTask : public PosixTask
{
public:
CopyTask(int sock, uint length);
virtual ~CopyTask();
bool run();
private:
CopyTask();
public:
CopyTask(int sock, uint length);
virtual ~CopyTask();
bool run();
private:
CopyTask();
};
}
} // namespace storagemanager

300
storage-manager/src/Downloader.cpp
View File

@ -27,209 +27,211 @@ using namespace std;
namespace bf = boost::filesystem;
namespace storagemanager
{
Downloader::Downloader() : maxDownloads(0)
{
storage = CloudStorage::get();
configListener();
Config::get()->addConfigListener(this);
workers.setName("Downloader");
logger = SMLogging::get();
tmpPath = "downloading";
bytesDownloaded = 0;
storage = CloudStorage::get();
configListener();
Config::get()->addConfigListener(this);
workers.setName("Downloader");
logger = SMLogging::get();
tmpPath = "downloading";
bytesDownloaded = 0;
}
Downloader::~Downloader()
{
Config::get()->removeConfigListener(this);
Config::get()->removeConfigListener(this);
}
void Downloader::download(const vector<const string *> &keys, vector<int> *errnos, vector<size_t> *sizes,
const bf::path &prefix, boost::mutex *cache_lock)
void Downloader::download(const vector<const string*>& keys, vector<int>* errnos, vector<size_t>* sizes,
const bf::path& prefix, boost::mutex* cache_lock)
{
uint counter = keys.size();
boost::condition condvar;
DownloadListener listener(&counter, &condvar);
vector<boost::shared_ptr<Download> > ownedDownloads(keys.size());
// the caller is holding cache_lock
/* if a key is already being downloaded, attach listener to that Download instance.
if it is not already being downloaded, make a new Download instance.
wait for the listener to tell us that it's done.
*/
boost::unique_lock<boost::mutex> s(lock);
for (uint i = 0; i < keys.size(); i++)
uint counter = keys.size();
boost::condition condvar;
DownloadListener listener(&counter, &condvar);
vector<boost::shared_ptr<Download> > ownedDownloads(keys.size());
// the caller is holding cache_lock
/* if a key is already being downloaded, attach listener to that Download instance.
if it is not already being downloaded, make a new Download instance.
wait for the listener to tell us that it's done.
*/
boost::unique_lock<boost::mutex> s(lock);
for (uint i = 0; i < keys.size(); i++)
{
boost::shared_ptr<Download> newDL(new Download(*keys[i], prefix, cache_lock, this));
auto it = downloads.find(newDL); // kinda sucks to have to search this way.
if (it == downloads.end())
{
boost::shared_ptr<Download> newDL(new Download(*keys[i], prefix, cache_lock, this));
auto it = downloads.find(newDL); // kinda sucks to have to search this way.
if (it == downloads.end())
{
newDL->listeners.push_back(&listener);
ownedDownloads[i] = newDL;
downloads.insert(newDL);
workers.addJob(newDL);
}
else
{
//assert((*it)->key == *keys[i]);
//cout << "Waiting for the existing download of " << *keys[i] << endl;
// a Download is technically in the map until all of the Downloads issued by its owner
// have finished. So, we have to test whether this existing download has already finished
// or not before waiting for it. We could have Downloads remove themselves on completion.
// TBD. Need to just get this working first.
if ((*it)->finished)
--counter;
else
(*it)->listeners.push_back(&listener);
}
newDL->listeners.push_back(&listener);
ownedDownloads[i] = newDL;
downloads.insert(newDL);
workers.addJob(newDL);
}
s.unlock();
// wait for the downloads to finish
while (counter > 0)
condvar.wait(*cache_lock);
// check success, gather sizes from downloads started by this thread
sizes->resize(keys.size());
errnos->resize(keys.size());
char buf[80];
s.lock();
for (uint i = 0; i < keys.size(); i++)
else
{
if (ownedDownloads[i])
{
assert(ownedDownloads[i]->finished);
(*sizes)[i] = ownedDownloads[i]->size;
(*errnos)[i] = ownedDownloads[i]->dl_errno;
if ((*errnos)[i])
logger->log(LOG_ERR, "Downloader: failed to download %s, got '%s'", keys[i]->c_str(),
// assert((*it)->key == *keys[i]);
// cout << "Waiting for the existing download of " << *keys[i] << endl;
// a Download is technically in the map until all of the Downloads issued by its owner
// have finished. So, we have to test whether this existing download has already finished
// or not before waiting for it. We could have Downloads remove themselves on completion.
// TBD. Need to just get this working first.
if ((*it)->finished)
--counter;
else
(*it)->listeners.push_back(&listener);
}
}
s.unlock();
// wait for the downloads to finish
while (counter > 0)
condvar.wait(*cache_lock);
// check success, gather sizes from downloads started by this thread
sizes->resize(keys.size());
errnos->resize(keys.size());
char buf[80];
s.lock();
for (uint i = 0; i < keys.size(); i++)
{
if (ownedDownloads[i])
{
assert(ownedDownloads[i]->finished);
(*sizes)[i] = ownedDownloads[i]->size;
(*errnos)[i] = ownedDownloads[i]->dl_errno;
if ((*errnos)[i])
logger->log(LOG_ERR, "Downloader: failed to download %s, got '%s'", keys[i]->c_str(),
strerror_r((*errnos)[i], buf, 80));
downloads.erase(ownedDownloads[i]);
bytesDownloaded += (*sizes)[i];
}
else
{
(*sizes)[i] = 0;
(*errnos)[i] = 0;
}
downloads.erase(ownedDownloads[i]);
bytesDownloaded += (*sizes)[i];
}
else
{
(*sizes)[i] = 0;
(*errnos)[i] = 0;
}
}
}
void Downloader::printKPIs() const
{
cout << "Downloader: bytesDownloaded = " << bytesDownloaded << endl;
cout << "Downloader: bytesDownloaded = " << bytesDownloaded << endl;
}
bool Downloader::inProgress(const string &key)
bool Downloader::inProgress(const string& key)
{
boost::shared_ptr<Download> tmp(new Download(key));
boost::unique_lock<boost::mutex> s(lock);
auto it = downloads.find(tmp);
if (it != downloads.end())
return !(*it)->finished;
return false;
boost::shared_ptr<Download> tmp(new Download(key));
boost::unique_lock<boost::mutex> s(lock);
auto it = downloads.find(tmp);
if (it != downloads.end())
return !(*it)->finished;
return false;
}
const bf::path & Downloader::getTmpPath() const
const bf::path& Downloader::getTmpPath() const
{
return tmpPath;
return tmpPath;
}
/* The helper fcns */
Downloader::Download::Download(const string &source, const bf::path &_dlPath, boost::mutex *_lock, Downloader *_dl) :
dlPath(_dlPath), key(source), dl_errno(0), size(0), lock(_lock), finished(false), itRan(false), dl(_dl)
Downloader::Download::Download(const string& source, const bf::path& _dlPath, boost::mutex* _lock,
Downloader* _dl)
: dlPath(_dlPath), key(source), dl_errno(0), size(0), lock(_lock), finished(false), itRan(false), dl(_dl)
{
}
Downloader::Download::Download(const string &source) :
key(source), dl_errno(0), size(0), lock(NULL), finished(false), itRan(false), dl(NULL)
Downloader::Download::Download(const string& source)
: key(source), dl_errno(0), size(0), lock(NULL), finished(false), itRan(false), dl(NULL)
{
}
Downloader::Download::~Download()
{
assert(!itRan || finished);
assert(!itRan || finished);
}
void Downloader::Download::operator()()
{
itRan = true;
CloudStorage *storage = CloudStorage::get();
// download to a tmp path
if (!bf::exists(dlPath / dl->getTmpPath()))
bf::create_directories(dlPath / dl->getTmpPath());
bf::path tmpFile = dlPath / dl->getTmpPath() / key;
int err = storage->getObject(key, tmpFile.string(), &size);
if (err != 0)
{
dl_errno = errno;
bf::remove(tmpFile);
size = 0;
}
// move it to its proper place
boost::system::error_code berr;
bf::rename(tmpFile, dlPath / key, berr);
if (berr)
{
dl_errno = berr.value();
bf::remove(tmpFile);
size = 0;
}
lock->lock();
finished = true;
for (uint i = 0; i < listeners.size(); i++)
listeners[i]->downloadFinished();
lock->unlock();
itRan = true;
CloudStorage* storage = CloudStorage::get();
// download to a tmp path
if (!bf::exists(dlPath / dl->getTmpPath()))
bf::create_directories(dlPath / dl->getTmpPath());
bf::path tmpFile = dlPath / dl->getTmpPath() / key;
int err = storage->getObject(key, tmpFile.string(), &size);
if (err != 0)
{
dl_errno = errno;
bf::remove(tmpFile);
size = 0;
}
// move it to its proper place
boost::system::error_code berr;
bf::rename(tmpFile, dlPath / key, berr);
if (berr)
{
dl_errno = berr.value();
bf::remove(tmpFile);
size = 0;
}
lock->lock();
finished = true;
for (uint i = 0; i < listeners.size(); i++)
listeners[i]->downloadFinished();
lock->unlock();
}
Downloader::DownloadListener::DownloadListener(uint *_counter, boost::condition *condvar) : counter(_counter), cond(condvar)
Downloader::DownloadListener::DownloadListener(uint* _counter, boost::condition* condvar)
: counter(_counter), cond(condvar)
{
}
void Downloader::DownloadListener::downloadFinished()
{
(*counter)--;
if ((*counter) == 0)
cond->notify_all();
(*counter)--;
if ((*counter) == 0)
cond->notify_all();
}
inline size_t Downloader::DLHasher::operator()(const boost::shared_ptr<Download> &d) const
inline size_t Downloader::DLHasher::operator()(const boost::shared_ptr<Download>& d) const
{
// since the keys start with a uuid, we can probably get away with just returning the first 8 chars
// or as a compromise, hashing only the first X chars. For later.
return hash<string>()(d->key);
// since the keys start with a uuid, we can probably get away with just returning the first 8 chars
// or as a compromise, hashing only the first X chars. For later.
return hash<string>()(d->key);
}
inline bool Downloader::DLEquals::operator()(const boost::shared_ptr<Download> &d1, const boost::shared_ptr<Download> &d2) const
inline bool Downloader::DLEquals::operator()(const boost::shared_ptr<Download>& d1,
const boost::shared_ptr<Download>& d2) const
{
return (d1->key == d2->key);
return (d1->key == d2->key);
}
void Downloader::configListener()
{
// Downloader threads
string stmp = Config::get()->getValue("ObjectStorage", "max_concurrent_downloads");
if (maxDownloads == 0)
maxDownloads = 20;
if (stmp.empty())
// Downloader threads
string stmp = Config::get()->getValue("ObjectStorage", "max_concurrent_downloads");
if (maxDownloads == 0)
maxDownloads = 20;
if (stmp.empty())
{
logger->log(LOG_CRIT, "max_concurrent_downloads is not set. Using current value = %u", maxDownloads);
}
try
{
uint newValue = stoul(stmp);
if (newValue != maxDownloads)
{
logger->log(LOG_CRIT, "max_concurrent_downloads is not set. Using current value = %u",maxDownloads);
}
try
{
uint newValue = stoul(stmp);
if (newValue != maxDownloads)
{
maxDownloads = newValue;
workers.setMaxThreads(maxDownloads);
logger->log(LOG_INFO, "max_concurrent_downloads = %u",maxDownloads);
}
}
catch (invalid_argument &)
{
logger->log(LOG_CRIT, "max_concurrent_downloads is not a number. Using current value = %u",maxDownloads);
maxDownloads = newValue;
workers.setMaxThreads(maxDownloads);
logger->log(LOG_INFO, "max_concurrent_downloads = %u", maxDownloads);
}
}
catch (invalid_argument&)
{
logger->log(LOG_CRIT, "max_concurrent_downloads is not a number. Using current value = %u", maxDownloads);
}
}
}
} // namespace storagemanager

142
storage-manager/src/Downloader.h
View File

@ -34,81 +34,79 @@
namespace storagemanager
{
class Downloader : public ConfigListener
{
public:
Downloader();
virtual ~Downloader();
// caller owns the memory for the strings.
// errors are reported through errnos
void download(const std::vector<const std::string *> &keys,
std::vector<int> *errnos, std::vector<size_t> *sizes, const boost::filesystem::path &prefix,
boost::mutex *lock);
bool inProgress(const std::string &); // call this holding the cache's lock
const boost::filesystem::path & getTmpPath() const;
void printKPIs() const;
public:
Downloader();
virtual ~Downloader();
virtual void configListener() override;
private:
uint maxDownloads;
//boost::filesystem::path downloadPath;
boost::mutex lock;
class DownloadListener
{
public:
DownloadListener(uint *counter, boost::condition *condvar);
void downloadFinished();
private:
uint *counter;
boost::condition *cond;
};
/* Possible optimization. Downloads used to use pointers to strings to avoid an extra copy.
Out of paranoid during debugging, I made it copy the strings instead for a clearer lifecycle.
However, it _should_ be safe to do.
*/
struct Download : public ThreadPool::Job
{
Download(const std::string &source, const boost::filesystem::path &_dlPath, boost::mutex *, Downloader *);
Download(const std::string &source);
virtual ~Download();
void operator()();
boost::filesystem::path dlPath;
const std::string key;
int dl_errno; // to propagate errors from the download job to the caller
size_t size;
boost::mutex *lock;
bool finished, itRan;
Downloader *dl;
std::vector<DownloadListener *> listeners;
};
struct DLHasher
{
size_t operator()(const boost::shared_ptr<Download> &d) const;
};
struct DLEquals
{
bool operator()(const boost::shared_ptr<Download> &d1, const boost::shared_ptr<Download> &d2) const;
};
typedef std::unordered_set<boost::shared_ptr<Download>, DLHasher, DLEquals> Downloads_t;
Downloads_t downloads;
boost::filesystem::path tmpPath;
ThreadPool workers;
CloudStorage *storage;
SMLogging *logger;
// caller owns the memory for the strings.
// errors are reported through errnos
void download(const std::vector<const std::string*>& keys, std::vector<int>* errnos,
std::vector<size_t>* sizes, const boost::filesystem::path& prefix, boost::mutex* lock);
bool inProgress(const std::string&); // call this holding the cache's lock
const boost::filesystem::path& getTmpPath() const;
// KPIs
size_t bytesDownloaded;
void printKPIs() const;
virtual void configListener() override;
private:
uint maxDownloads;
// boost::filesystem::path downloadPath;
boost::mutex lock;
class DownloadListener
{
public:
DownloadListener(uint* counter, boost::condition* condvar);
void downloadFinished();
private:
uint* counter;
boost::condition* cond;
};
/* Possible optimization. Downloads used to use pointers to strings to avoid an extra copy.
Out of paranoid during debugging, I made it copy the strings instead for a clearer lifecycle.
However, it _should_ be safe to do.
*/
struct Download : public ThreadPool::Job
{
Download(const std::string& source, const boost::filesystem::path& _dlPath, boost::mutex*, Downloader*);
Download(const std::string& source);
virtual ~Download();
void operator()();
boost::filesystem::path dlPath;
const std::string key;
int dl_errno; // to propagate errors from the download job to the caller
size_t size;
boost::mutex* lock;
bool finished, itRan;
Downloader* dl;
std::vector<DownloadListener*> listeners;
};
struct DLHasher
{
size_t operator()(const boost::shared_ptr<Download>& d) const;
};
struct DLEquals
{
bool operator()(const boost::shared_ptr<Download>& d1, const boost::shared_ptr<Download>& d2) const;
};
typedef std::unordered_set<boost::shared_ptr<Download>, DLHasher, DLEquals> Downloads_t;
Downloads_t downloads;
boost::filesystem::path tmpPath;
ThreadPool workers;
CloudStorage* storage;
SMLogging* logger;
// KPIs
size_t bytesDownloaded;
};
}
} // namespace storagemanager

2703
storage-manager/src/IOCoordinator.cpp
View File

File diff suppressed because it is too large Load Diff

170
storage-manager/src/IOCoordinator.h
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <sys/types.h>
@ -38,97 +37,94 @@
namespace storagemanager
{
boost::shared_array<char> seekToEndOfHeader1(int fd, size_t *bytesRead);
boost::shared_array<char> seekToEndOfHeader1(int fd, size_t* bytesRead);
class IOCoordinator : public boost::noncopyable
{
public:
static IOCoordinator *get();
virtual ~IOCoordinator();
public:
static IOCoordinator* get();
virtual ~IOCoordinator();
ssize_t read(const char *filename, uint8_t *data, off_t offset, size_t length);
ssize_t write(const char *filename, const uint8_t *data, off_t offset, size_t length);
ssize_t append(const char *filename, const uint8_t *data, size_t length);
int open(const char *filename, int openmode, struct stat *out);
int listDirectory(const char *filename, std::vector<std::string> *listing);
int stat(const char *path, struct stat *out);
int truncate(const char *path, size_t newsize);
int unlink(const char *path);
int copyFile(const char *filename1, const char *filename2);
// The shared logic for merging a journal file with its base file.
// len should be set to the length of the data requested
boost::shared_array<uint8_t> mergeJournal(const char *objectPath, const char *journalPath, off_t offset,
size_t len, size_t *sizeRead) const;
// this version modifies object data in memory, given the journal filename. Processes the whole object
// and whole journal file.
int mergeJournalInMem(boost::shared_array<uint8_t> &objData, size_t len, const char *journalPath,
size_t *sizeRead) const;
// this version of MJIM has a higher IOPS requirement and lower mem usage.
int mergeJournalInMem_bigJ(boost::shared_array<uint8_t> &objData, size_t len, const char *journalPath,
size_t *sizeRead) const;
// this version takes already-open file descriptors, and an already-allocated buffer as input.
// file descriptor are positioned, eh, best not to assume anything about their positions
// on return.
// Not implemented yet. At the point of this writing, we will use the existing versions, even though
// it's wasteful, and will leave this as a likely future optimization.
int mergeJournal(int objFD, int journalFD, uint8_t *buf, off_t offset, size_t *len) const;
/* Lock manipulation fcns. They can lock on any param given to them. For convention's sake,
the parameter should mostly be the abs filename being accessed. */
void readLock(const std::string &filename);
void writeLock(const std::string &filename);
void readUnlock(const std::string &filename);
void writeUnlock(const std::string &filename);
// some accessors...
const boost::filesystem::path &getCachePath() const;
const boost::filesystem::path &getJournalPath() const;
const boost::filesystem::path &getMetadataPath() const;
void printKPIs() const;
private:
IOCoordinator();
Config *config;
Cache *cache;
SMLogging *logger;
Replicator *replicator;
Ownership ownership; // ACK! Need a new name for this!
ssize_t read(const char* filename, uint8_t* data, off_t offset, size_t length);
ssize_t write(const char* filename, const uint8_t* data, off_t offset, size_t length);
ssize_t append(const char* filename, const uint8_t* data, size_t length);
int open(const char* filename, int openmode, struct stat* out);
int listDirectory(const char* filename, std::vector<std::string>* listing);
int stat(const char* path, struct stat* out);
int truncate(const char* path, size_t newsize);
int unlink(const char* path);
int copyFile(const char* filename1, const char* filename2);
size_t objectSize;
boost::filesystem::path journalPath;
boost::filesystem::path cachePath;
boost::filesystem::path metaPath;
std::map<std::string, RWLock *> locks;
boost::mutex lockMutex; // lol
void remove(const boost::filesystem::path &path);
void deleteMetaFile(const boost::filesystem::path &file);
// The shared logic for merging a journal file with its base file.
// len should be set to the length of the data requested
boost::shared_array<uint8_t> mergeJournal(const char* objectPath, const char* journalPath, off_t offset,
size_t len, size_t* sizeRead) const;
int _truncate(const boost::filesystem::path &path, size_t newsize, ScopedFileLock *lock);
ssize_t _write(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length,
const boost::filesystem::path &firstDir);
int loadObjectAndJournal(const char *objFilename, const char *journalFilename,
uint8_t *data, off_t offset, size_t length);
int loadObject(int fd, uint8_t *data, off_t offset, size_t length);
// some KPIs
// from the user's POV...
size_t bytesRead, bytesWritten, filesOpened, filesCreated, filesCopied;
size_t filesDeleted, bytesCopied, filesTruncated, listingCount, callsToWrite;
// from IOC's pov...
size_t iocFilesOpened, iocObjectsCreated, iocJournalsCreated, iocFilesDeleted;
size_t iocBytesRead, iocBytesWritten;
// this version modifies object data in memory, given the journal filename. Processes the whole object
// and whole journal file.
int mergeJournalInMem(boost::shared_array<uint8_t>& objData, size_t len, const char* journalPath,
size_t* sizeRead) const;
// this version of MJIM has a higher IOPS requirement and lower mem usage.
int mergeJournalInMem_bigJ(boost::shared_array<uint8_t>& objData, size_t len, const char* journalPath,
size_t* sizeRead) const;
// this version takes already-open file descriptors, and an already-allocated buffer as input.
// file descriptor are positioned, eh, best not to assume anything about their positions
// on return.
// Not implemented yet. At the point of this writing, we will use the existing versions, even though
// it's wasteful, and will leave this as a likely future optimization.
int mergeJournal(int objFD, int journalFD, uint8_t* buf, off_t offset, size_t* len) const;
/* Lock manipulation fcns. They can lock on any param given to them. For convention's sake,
the parameter should mostly be the abs filename being accessed. */
void readLock(const std::string& filename);
void writeLock(const std::string& filename);
void readUnlock(const std::string& filename);
void writeUnlock(const std::string& filename);
// some accessors...
const boost::filesystem::path& getCachePath() const;
const boost::filesystem::path& getJournalPath() const;
const boost::filesystem::path& getMetadataPath() const;
void printKPIs() const;
private:
IOCoordinator();
Config* config;
Cache* cache;
SMLogging* logger;
Replicator* replicator;
Ownership ownership; // ACK! Need a new name for this!
size_t objectSize;
boost::filesystem::path journalPath;
boost::filesystem::path cachePath;
boost::filesystem::path metaPath;
std::map<std::string, RWLock*> locks;
boost::mutex lockMutex; // lol
void remove(const boost::filesystem::path& path);
void deleteMetaFile(const boost::filesystem::path& file);
int _truncate(const boost::filesystem::path& path, size_t newsize, ScopedFileLock* lock);
ssize_t _write(const boost::filesystem::path& filename, const uint8_t* data, off_t offset, size_t length,
const boost::filesystem::path& firstDir);
int loadObjectAndJournal(const char* objFilename, const char* journalFilename, uint8_t* data, off_t offset,
size_t length);
int loadObject(int fd, uint8_t* data, off_t offset, size_t length);
// some KPIs
// from the user's POV...
size_t bytesRead, bytesWritten, filesOpened, filesCreated, filesCopied;
size_t filesDeleted, bytesCopied, filesTruncated, listingCount, callsToWrite;
// from IOC's pov...
size_t iocFilesOpened, iocObjectsCreated, iocJournalsCreated, iocFilesDeleted;
size_t iocBytesRead, iocBytesWritten;
};
}
} // namespace storagemanager

194
storage-manager/src/ListDirectoryTask.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "ListDirectoryTask.h"
#include "SMLogging.h"
#include "messageFormat.h"
@ -27,7 +26,6 @@ using namespace std;
namespace storagemanager
{
ListDirectoryTask::ListDirectoryTask(int sock, uint len) : PosixTask(sock, len)
{
}
@ -37,108 +35,110 @@ ListDirectoryTask::~ListDirectoryTask()
}
#define check_error(msg, ret) \
if (!success) \
{ \
handleError(msg, errno); \
return ret; \
}
if (!success) \
{ \
handleError(msg, errno); \
return ret; \
}
#define min(x, y) (x < y ? x : y)
bool ListDirectoryTask::writeString(uint8_t *buf, int *offset, int size, const string &str)
bool ListDirectoryTask::writeString(uint8_t* buf, int* offset, int size, const string& str)
{
bool success;
if (size - *offset < 4) // eh, let's not frag 4 bytes.
bool success;
if (size - *offset < 4) // eh, let's not frag 4 bytes.
{
success = write(buf, *offset);
check_error("ListDirectoryTask::writeString()", false);
*offset = 0;
}
int count = 0, len = str.length();
*((uint32_t*)&buf[*offset]) = len;
*offset += 4;
while (count < len)
{
int toWrite = min(size - *offset, len);
memcpy(&buf[*offset], &str.data()[count], toWrite);
count += toWrite;
*offset += toWrite;
if (*offset == size)
{
success = write(buf, *offset);
check_error("ListDirectoryTask::writeString()", false);
*offset = 0;
success = write(buf, *offset);
check_error("ListDirectoryTask::writeString()", false);
*offset = 0;
}
int count = 0, len = str.length();
*((uint32_t *) &buf[*offset]) = len;
*offset += 4;
while (count < len)
{
int toWrite = min(size - *offset, len);
memcpy(&buf[*offset], &str.data()[count], toWrite);
count += toWrite;
*offset += toWrite;
if (*offset == size)
{
success = write(buf, *offset);
check_error("ListDirectoryTask::writeString()", false);
*offset = 0;
}
}
return true;
}
return true;
}
bool ListDirectoryTask::run()
{
SMLogging* logger = SMLogging::get();
bool success;
uint8_t buf[1024] = {0};
int err;
if (getLength() > 1023) {
handleError("ListDirectoryTask read", ENAMETOOLONG);
return true;
}
err = read(buf, getLength());
if (err<0)
{
handleError("ListDirectoryTask read", errno);
return false;
}
listdir_cmd *cmd = (listdir_cmd *) buf;
#ifdef SM_TRACE
logger->log(LOG_DEBUG,"list_directory %s.",cmd->path);
#endif
vector<string> listing;
try {
err = ioc->listDirectory(cmd->path, &listing);
}
catch (exception &e)
{
logger->log(LOG_ERR, "ListDirectoryTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err)
{
handleError("ListDirectory", errno);
return true;
}
// be careful modifying the listdir return types...
uint payloadLen = sizeof(listdir_resp) + (sizeof(listdir_resp_entry) * listing.size());
for (uint i = 0; i < listing.size(); i++)
payloadLen += listing[i].size();
sm_response *resp = (sm_response *) buf;
resp->header.type = SM_MSG_START;
resp->header.payloadLen = payloadLen + sizeof(sm_response) - sizeof(sm_msg_header);
resp->header.flags = 0;
resp->returnCode = 0;
listdir_resp *r = (listdir_resp *) resp->payload;
r->elements = listing.size();
int offset = (uint64_t) r->entries - (uint64_t) buf;
for (uint i = 0; i < listing.size(); i++)
{
success = writeString(buf, &offset, 1024, listing[i]);
check_error("ListDirectoryTask write", false);
}
if (offset != 0)
{
success = write(buf, offset);
check_error("ListDirectoryTask write", false);
}
SMLogging* logger = SMLogging::get();
bool success;
uint8_t buf[1024] = {0};
int err;
if (getLength() > 1023)
{
handleError("ListDirectoryTask read", ENAMETOOLONG);
return true;
}
err = read(buf, getLength());
if (err < 0)
{
handleError("ListDirectoryTask read", errno);
return false;
}
listdir_cmd* cmd = (listdir_cmd*)buf;
#ifdef SM_TRACE
logger->log(LOG_DEBUG, "list_directory %s.", cmd->path);
#endif
vector<string> listing;
try
{
err = ioc->listDirectory(cmd->path, &listing);
}
catch (exception& e)
{
logger->log(LOG_ERR, "ListDirectoryTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err)
{
handleError("ListDirectory", errno);
return true;
}
// be careful modifying the listdir return types...
uint payloadLen = sizeof(listdir_resp) + (sizeof(listdir_resp_entry) * listing.size());
for (uint i = 0; i < listing.size(); i++)
payloadLen += listing[i].size();
sm_response* resp = (sm_response*)buf;
resp->header.type = SM_MSG_START;
resp->header.payloadLen = payloadLen + sizeof(sm_response) - sizeof(sm_msg_header);
resp->header.flags = 0;
resp->returnCode = 0;
listdir_resp* r = (listdir_resp*)resp->payload;
r->elements = listing.size();
int offset = (uint64_t)r->entries - (uint64_t)buf;
for (uint i = 0; i < listing.size(); i++)
{
success = writeString(buf, &offset, 1024, listing[i]);
check_error("ListDirectoryTask write", false);
}
if (offset != 0)
{
success = write(buf, offset);
check_error("ListDirectoryTask write", false);
}
return true;
}
}
} // namespace storagemanager

25
storage-manager/src/ListDirectoryTask.h
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "PosixTask.h"
@ -23,20 +22,18 @@
namespace storagemanager
{
class ListDirectoryTask : public PosixTask
{
public:
ListDirectoryTask(int sock, uint length);
virtual ~ListDirectoryTask();
bool run();
private:
ListDirectoryTask();
bool writeString(uint8_t *buf, int *offset, int size, const std::string &str);
public:
ListDirectoryTask(int sock, uint length);
virtual ~ListDirectoryTask();
bool run();
private:
ListDirectoryTask();
bool writeString(uint8_t* buf, int* offset, int size, const std::string& str);
};
}
} // namespace storagemanager

377
storage-manager/src/LocalStorage.cpp
View File

@ -15,8 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <boost/filesystem.hpp>
#include <iostream>
#include <sys/types.h>
@ -31,41 +29,40 @@ namespace bf = boost::filesystem;
namespace storagemanager
{
LocalStorage::LocalStorage()
{
prefix = Config::get()->getValue("LocalStorage", "path");
//cout << "LS: got prefix " << prefix << endl;
if (!bf::is_directory(prefix))
prefix = Config::get()->getValue("LocalStorage", "path");
// cout << "LS: got prefix " << prefix << endl;
if (!bf::is_directory(prefix))
{
try
{
try
{
bf::create_directories(prefix);
}
catch (exception &e)
{
logger->log(LOG_CRIT, "Failed to create %s, got: %s", prefix.string().c_str(), e.what());
throw e;
}
bf::create_directories(prefix);
}
string stmp = Config::get()->getValue("LocalStorage", "fake_latency");
if (!stmp.empty() && (stmp[0] == 'Y' || stmp[0] == 'y'))
catch (exception& e)
{
fakeLatency = true;
stmp = Config::get()->getValue("LocalStorage", "max_latency");
usecLatencyCap = strtoull(stmp.c_str(), NULL, 10);
if (usecLatencyCap == 0)
{
logger->log(LOG_CRIT, "LocalStorage: bad value for max_latency");
throw runtime_error("LocalStorage: bad value for max_latency");
}
r_seed = (uint) ::time(NULL);
logger->log(LOG_DEBUG, "LocalStorage: Will simulate cloud latency of max %llu us", usecLatencyCap);
logger->log(LOG_CRIT, "Failed to create %s, got: %s", prefix.string().c_str(), e.what());
throw e;
}
else
fakeLatency = false;
bytesRead = bytesWritten = 0;
}
string stmp = Config::get()->getValue("LocalStorage", "fake_latency");
if (!stmp.empty() && (stmp[0] == 'Y' || stmp[0] == 'y'))
{
fakeLatency = true;
stmp = Config::get()->getValue("LocalStorage", "max_latency");
usecLatencyCap = strtoull(stmp.c_str(), NULL, 10);
if (usecLatencyCap == 0)
{
logger->log(LOG_CRIT, "LocalStorage: bad value for max_latency");
throw runtime_error("LocalStorage: bad value for max_latency");
}
r_seed = (uint)::time(NULL);
logger->log(LOG_DEBUG, "LocalStorage: Will simulate cloud latency of max %llu us", usecLatencyCap);
}
else
fakeLatency = false;
bytesRead = bytesWritten = 0;
}
LocalStorage::~LocalStorage()
@ -74,195 +71,199 @@ LocalStorage::~LocalStorage()
void LocalStorage::printKPIs() const
{
cout << "LocalStorage" << endl;
cout << "\tbytesRead = " << bytesRead << endl;
cout << "\tbytesWritten = " << bytesWritten << endl;
CloudStorage::printKPIs();
cout << "LocalStorage" << endl;
cout << "\tbytesRead = " << bytesRead << endl;
cout << "\tbytesWritten = " << bytesWritten << endl;
CloudStorage::printKPIs();
}
const bf::path & LocalStorage::getPrefix() const
const bf::path& LocalStorage::getPrefix() const
{
return prefix;
return prefix;
}
inline void LocalStorage::addLatency()
{
if (fakeLatency)
{
uint64_t usec_delay = ((double) rand_r(&r_seed) / (double) RAND_MAX) * usecLatencyCap;
::usleep(usec_delay);
}
if (fakeLatency)
{
uint64_t usec_delay = ((double)rand_r(&r_seed) / (double)RAND_MAX) * usecLatencyCap;
::usleep(usec_delay);
}
}
int LocalStorage::copy(const bf::path &source, const bf::path &dest)
int LocalStorage::copy(const bf::path& source, const bf::path& dest)
{
boost::system::error_code err;
bf::copy_file(source, dest, bf::copy_option::fail_if_exists, err);
if (err)
{
errno = err.value();
::unlink(dest.string().c_str());
return -1;
}
return 0;
boost::system::error_code err;
bf::copy_file(source, dest, bf::copy_option::fail_if_exists, err);
if (err)
{
errno = err.value();
::unlink(dest.string().c_str());
return -1;
}
return 0;
}
bf::path operator+(const bf::path &p1, const bf::path &p2)
bf::path operator+(const bf::path& p1, const bf::path& p2)
{
bf::path ret(p1);
ret /= p2;
bf::path ret(p1);
ret /= p2;
return ret;
}
int LocalStorage::getObject(const string& source, const string& dest, size_t* size)
{
addLatency();
int ret = copy(prefix / source, dest);
if (ret)
return ret;
size_t _size = bf::file_size(dest);
if (size)
*size = _size;
bytesRead += _size;
bytesWritten += _size;
++objectsGotten;
return ret;
}
int LocalStorage::getObject(const string &source, const string &dest, size_t *size)
int LocalStorage::getObject(const std::string& sourceKey, boost::shared_array<uint8_t>* data, size_t* size)
{
addLatency();
addLatency();
int ret = copy(prefix / source, dest);
if (ret)
return ret;
size_t _size = bf::file_size(dest);
if (size)
*size = _size;
bf::path source = prefix / sourceKey;
const char* c_source = source.string().c_str();
// char buf[80];
int l_errno;
int fd = ::open(c_source, O_RDONLY);
if (fd < 0)
{
l_errno = errno;
// logger->log(LOG_WARNING, "LocalStorage::getObject() failed to open %s, got '%s'", c_source,
// strerror_r(errno, buf, 80));
errno = l_errno;
return fd;
}
size_t l_size = bf::file_size(source);
data->reset(new uint8_t[l_size]);
size_t count = 0;
while (count < l_size)
{
int err = ::read(fd, &(*data)[count], l_size - count);
if (err < 0)
{
l_errno = errno;
// logger->log(LOG_WARNING, "LocalStorage::getObject() failed to read %s, got '%s'", c_source,
// strerror_r(errno, buf, 80));
close(fd);
bytesRead += count;
errno = l_errno;
return err;
}
count += err;
}
if (size)
*size = l_size;
close(fd);
bytesRead += l_size;
++objectsGotten;
return 0;
}
int LocalStorage::putObject(const string& source, const string& dest)
{
addLatency();
int ret = copy(source, prefix / dest);
if (ret == 0)
{
size_t _size = bf::file_size(source);
bytesRead += _size;
bytesWritten += _size;
++objectsGotten;
return ret;
}
int LocalStorage::getObject(const std::string &sourceKey, boost::shared_array<uint8_t> *data, size_t *size)
{
addLatency();
bf::path source = prefix / sourceKey;
const char *c_source = source.string().c_str();
//char buf[80];
int l_errno;
int fd = ::open(c_source, O_RDONLY);
if (fd < 0)
{
l_errno = errno;
//logger->log(LOG_WARNING, "LocalStorage::getObject() failed to open %s, got '%s'", c_source, strerror_r(errno, buf, 80));
errno = l_errno;
return fd;
}
size_t l_size = bf::file_size(source);
data->reset(new uint8_t[l_size]);
size_t count = 0;
while (count < l_size)
{
int err = ::read(fd, &(*data)[count], l_size - count);
if (err < 0)
{
l_errno = errno;
//logger->log(LOG_WARNING, "LocalStorage::getObject() failed to read %s, got '%s'", c_source, strerror_r(errno, buf, 80));
close(fd);
bytesRead += count;
errno = l_errno;
return err;
}
count += err;
}
if (size)
*size = l_size;
close(fd);
bytesRead += l_size;
++objectsGotten;
return 0;
}
int LocalStorage::putObject(const string &source, const string &dest)
{
addLatency();
int ret = copy(source, prefix / dest);
if (ret == 0)
{
size_t _size = bf::file_size(source);
bytesRead += _size;
bytesWritten += _size;
++objectsPut;
}
return ret;
}
int LocalStorage::putObject(boost::shared_array<uint8_t> data, size_t len, const string &dest)
{
addLatency();
bf::path destPath = prefix / dest;
const char *c_dest = destPath.string().c_str();
//char buf[80];
int l_errno;
int fd = ::open(c_dest, O_WRONLY | O_CREAT | O_TRUNC, 0600);
if (fd < 0)
{
l_errno = errno;
//logger->log(LOG_CRIT, "LocalStorage::putObject(): Failed to open %s, got '%s'", c_dest, strerror_r(errno, buf, 80));
errno = l_errno;
return fd;
}
size_t count = 0;
int err;
while (count < len)
{
err = ::write(fd, &data[count], len - count);
if (err < 0)
{
l_errno = errno;
//logger->log(LOG_CRIT, "LocalStorage::putObject(): Failed to write to %s, got '%s'", c_dest, strerror_r(errno, buf, 80));
close(fd);
::unlink(c_dest);
errno = l_errno;
bytesWritten += count;
return err;
}
count += err;
}
close(fd);
bytesWritten += count;
++objectsPut;
return 0;
}
return ret;
}
int LocalStorage::copyObject(const string &source, const string &dest)
int LocalStorage::putObject(boost::shared_array<uint8_t> data, size_t len, const string& dest)
{
addLatency();
int ret = copy(prefix / source, prefix / dest);
if (ret == 0)
addLatency();
bf::path destPath = prefix / dest;
const char* c_dest = destPath.string().c_str();
// char buf[80];
int l_errno;
int fd = ::open(c_dest, O_WRONLY | O_CREAT | O_TRUNC, 0600);
if (fd < 0)
{
l_errno = errno;
// logger->log(LOG_CRIT, "LocalStorage::putObject(): Failed to open %s, got '%s'", c_dest,
// strerror_r(errno, buf, 80));
errno = l_errno;
return fd;
}
size_t count = 0;
int err;
while (count < len)
{
err = ::write(fd, &data[count], len - count);
if (err < 0)
{
++objectsCopied;
size_t _size = bf::file_size(prefix/source);
bytesRead += _size;
bytesWritten += _size;
l_errno = errno;
// logger->log(LOG_CRIT, "LocalStorage::putObject(): Failed to write to %s, got '%s'", c_dest,
// strerror_r(errno, buf, 80));
close(fd);
::unlink(c_dest);
errno = l_errno;
bytesWritten += count;
return err;
}
return ret;
count += err;
}
close(fd);
bytesWritten += count;
++objectsPut;
return 0;
}
int LocalStorage::deleteObject(const string &key)
int LocalStorage::copyObject(const string& source, const string& dest)
{
addLatency();
++objectsDeleted;
boost::system::error_code err;
bf::remove(prefix / key, err);
return 0;
addLatency();
int ret = copy(prefix / source, prefix / dest);
if (ret == 0)
{
++objectsCopied;
size_t _size = bf::file_size(prefix / source);
bytesRead += _size;
bytesWritten += _size;
}
return ret;
}
int LocalStorage::exists(const std::string &key, bool *out)
int LocalStorage::deleteObject(const string& key)
{
addLatency();
++existenceChecks;
*out = bf::exists(prefix / key);
return 0;
addLatency();
++objectsDeleted;
boost::system::error_code err;
bf::remove(prefix / key, err);
return 0;
}
int LocalStorage::exists(const std::string& key, bool* out)
{
addLatency();
++existenceChecks;
*out = bf::exists(prefix / key);
return 0;
}
} // namespace storagemanager

56
storage-manager/src/LocalStorage.h
View File

@ -24,37 +24,35 @@
namespace storagemanager
{
class LocalStorage : public CloudStorage
{
public:
LocalStorage();
virtual ~LocalStorage();
public:
LocalStorage();
virtual ~LocalStorage();
int getObject(const std::string &sourceKey, const std::string &destFile, size_t *size = NULL);
int getObject(const std::string &sourceKey, boost::shared_array<uint8_t> *data, size_t *size = NULL);
int putObject(const std::string &sourceFile, const std::string &destKey);
int putObject(const boost::shared_array<uint8_t> data, size_t len, const std::string &destKey);
int deleteObject(const std::string &key);
int copyObject(const std::string &sourceKey, const std::string &destKey);
int exists(const std::string &key, bool *out);
const boost::filesystem::path & getPrefix() const;
void printKPIs() const;
protected:
size_t bytesRead, bytesWritten;
private:
boost::filesystem::path prefix;
int copy(const boost::filesystem::path &sourceKey, const boost::filesystem::path &destKey);
// stuff for faking the latency on cloud ops
bool fakeLatency;
uint64_t usecLatencyCap;
uint r_seed;
void addLatency();
int getObject(const std::string& sourceKey, const std::string& destFile, size_t* size = NULL);
int getObject(const std::string& sourceKey, boost::shared_array<uint8_t>* data, size_t* size = NULL);
int putObject(const std::string& sourceFile, const std::string& destKey);
int putObject(const boost::shared_array<uint8_t> data, size_t len, const std::string& destKey);
int deleteObject(const std::string& key);
int copyObject(const std::string& sourceKey, const std::string& destKey);
int exists(const std::string& key, bool* out);
const boost::filesystem::path& getPrefix() const;
void printKPIs() const;
protected:
size_t bytesRead, bytesWritten;
private:
boost::filesystem::path prefix;
int copy(const boost::filesystem::path& sourceKey, const boost::filesystem::path& destKey);
// stuff for faking the latency on cloud ops
bool fakeLatency;
uint64_t usecLatencyCap;
uint r_seed;
void addLatency();
};
}
} // namespace storagemanager

781
storage-manager/src/MetadataFile.cpp
View File

@ -37,157 +37,154 @@ namespace bpt = boost::property_tree;
namespace bf = boost::filesystem;
namespace
{
boost::mutex mdfLock;
storagemanager::MetadataFile::MetadataConfig *inst = NULL;
uint64_t metadataFilesAccessed = 0;
}
{
boost::mutex mdfLock;
storagemanager::MetadataFile::MetadataConfig* inst = NULL;
uint64_t metadataFilesAccessed = 0;
} // namespace
namespace storagemanager
{
MetadataFile::MetadataConfig * MetadataFile::MetadataConfig::get()
MetadataFile::MetadataConfig* MetadataFile::MetadataConfig::get()
{
if (inst)
return inst;
boost::unique_lock<boost::mutex> s(mdfLock);
if (inst)
return inst;
inst = new MetadataConfig();
if (inst)
return inst;
boost::unique_lock<boost::mutex> s(mdfLock);
if (inst)
return inst;
inst = new MetadataConfig();
return inst;
}
MetadataFile::MetadataConfig::MetadataConfig()
{
Config *config = Config::get();
SMLogging *logger = SMLogging::get();
try
{
mObjectSize = stoul(config->getValue("ObjectStorage", "object_size"));
}
catch (...)
{
logger->log(LOG_CRIT, "ObjectStorage/object_size must be set to a numeric value");
throw runtime_error("Please set ObjectStorage/object)size in the storagemanager.cnf file");
}
try
{
msMetadataPath = config->getValue("ObjectStorage", "metadata_path");
if (msMetadataPath.empty())
{
logger->log(LOG_CRIT, "ObjectStorage/metadata_path is not set");
throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file");
}
}
catch (...)
{
logger->log(LOG_CRIT, "ObjectStorage/metadata_path is not set");
throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file");
}
Config* config = Config::get();
SMLogging* logger = SMLogging::get();
try
{
boost::filesystem::create_directories(msMetadataPath);
}
catch (exception &e)
{
logger->log(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what());
throw e;
}
try
{
mObjectSize = stoul(config->getValue("ObjectStorage", "object_size"));
}
catch (...)
{
logger->log(LOG_CRIT, "ObjectStorage/object_size must be set to a numeric value");
throw runtime_error("Please set ObjectStorage/object)size in the storagemanager.cnf file");
}
try
{
msMetadataPath = config->getValue("ObjectStorage", "metadata_path");
if (msMetadataPath.empty())
{
logger->log(LOG_CRIT, "ObjectStorage/metadata_path is not set");
throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file");
}
}
catch (...)
{
logger->log(LOG_CRIT, "ObjectStorage/metadata_path is not set");
throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file");
}
try
{
boost::filesystem::create_directories(msMetadataPath);
}
catch (exception& e)
{
logger->log(LOG_CRIT, "Failed to create %s, got: %s", msMetadataPath.c_str(), e.what());
throw e;
}
}
MetadataFile::MetadataFile()
{
mpConfig = MetadataConfig::get();
mpLogger = SMLogging::get();
mVersion=1;
mRevision=1;
_exists = false;
mpConfig = MetadataConfig::get();
mpLogger = SMLogging::get();
mVersion = 1;
mRevision = 1;
_exists = false;
}
MetadataFile::MetadataFile(const boost::filesystem::path &filename)
MetadataFile::MetadataFile(const boost::filesystem::path& filename)
{
mpConfig = MetadataConfig::get();
mpLogger = SMLogging::get();
mpConfig = MetadataConfig::get();
mpLogger = SMLogging::get();
_exists = true;
mFilename = mpConfig->msMetadataPath / (filename.string() + ".meta");
boost::unique_lock<boost::mutex> s(jsonCache.getMutex());
jsontree = jsonCache.get(mFilename);
if (!jsontree)
{
if (boost::filesystem::exists(mFilename))
{
jsontree.reset(new bpt::ptree());
boost::property_tree::read_json(mFilename.string(), *jsontree);
jsonCache.put(mFilename, jsontree);
s.unlock();
mVersion = 1;
mRevision = jsontree->get<int>("revision");
}
else
{
mVersion = 1;
mRevision = 1;
makeEmptyJsonTree();
s.unlock();
writeMetadata();
}
}
else
{
s.unlock();
mVersion = 1;
mRevision = jsontree->get<int>("revision");
}
++metadataFilesAccessed;
}
MetadataFile::MetadataFile(const boost::filesystem::path& filename, no_create_t, bool appendExt)
{
mpConfig = MetadataConfig::get();
mpLogger = SMLogging::get();
mFilename = filename;
if (appendExt)
mFilename = mpConfig->msMetadataPath / (mFilename.string() + ".meta");
boost::unique_lock<boost::mutex> s(jsonCache.getMutex());
jsontree = jsonCache.get(mFilename);
if (!jsontree)
{
if (boost::filesystem::exists(mFilename))
{
_exists = true;
jsontree.reset(new bpt::ptree());
boost::property_tree::read_json(mFilename.string(), *jsontree);
jsonCache.put(mFilename, jsontree);
s.unlock();
mVersion = 1;
mRevision = jsontree->get<int>("revision");
}
else
{
mVersion = 1;
mRevision = 1;
_exists = false;
makeEmptyJsonTree();
}
}
else
{
s.unlock();
_exists = true;
mFilename = mpConfig->msMetadataPath / (filename.string() + ".meta");
boost::unique_lock<boost::mutex> s(jsonCache.getMutex());
jsontree = jsonCache.get(mFilename);
if (!jsontree)
{
if (boost::filesystem::exists(mFilename))
{
jsontree.reset(new bpt::ptree());
boost::property_tree::read_json(mFilename.string(), *jsontree);
jsonCache.put(mFilename, jsontree);
s.unlock();
mVersion = 1;
mRevision = jsontree->get<int>("revision");
}
else
{
mVersion = 1;
mRevision = 1;
makeEmptyJsonTree();
s.unlock();
writeMetadata();
}
}
else
{
s.unlock();
mVersion = 1;
mRevision = jsontree->get<int>("revision");
}
++metadataFilesAccessed;
}
MetadataFile::MetadataFile(const boost::filesystem::path &filename, no_create_t,bool appendExt)
{
mpConfig = MetadataConfig::get();
mpLogger = SMLogging::get();
mFilename = filename;
if(appendExt)
mFilename = mpConfig->msMetadataPath /(mFilename.string() + ".meta");
boost::unique_lock<boost::mutex> s(jsonCache.getMutex());
jsontree = jsonCache.get(mFilename);
if (!jsontree)
{
if (boost::filesystem::exists(mFilename))
{
_exists = true;
jsontree.reset(new bpt::ptree());
boost::property_tree::read_json(mFilename.string(), *jsontree);
jsonCache.put(mFilename, jsontree);
s.unlock();
mVersion = 1;
mRevision = jsontree->get<int>("revision");
}
else
{
mVersion = 1;
mRevision = 1;
_exists = false;
makeEmptyJsonTree();
}
}
else
{
s.unlock();
_exists = true;
mVersion = 1;
mRevision = jsontree->get<int>("revision");
}
++metadataFilesAccessed;
mVersion = 1;
mRevision = jsontree->get<int>("revision");
}
++metadataFilesAccessed;
}
MetadataFile::~MetadataFile()
@ -196,381 +193,381 @@ MetadataFile::~MetadataFile()
void MetadataFile::makeEmptyJsonTree()
{
jsontree.reset(new bpt::ptree());
boost::property_tree::ptree objs;
jsontree->put("version",mVersion);
jsontree->put("revision",mRevision);
jsontree->add_child("objects", objs);
jsontree.reset(new bpt::ptree());
boost::property_tree::ptree objs;
jsontree->put("version", mVersion);
jsontree->put("revision", mRevision);
jsontree->add_child("objects", objs);
}
void MetadataFile::printKPIs()
{
cout << "Metadata files accessed = " << metadataFilesAccessed << endl;
cout << "Metadata files accessed = " << metadataFilesAccessed << endl;
}
int MetadataFile::stat(struct stat *out) const
int MetadataFile::stat(struct stat* out) const
{
int err = ::stat(mFilename.c_str(), out);
if (err)
return err;
out->st_size = getLength();
return 0;
int err = ::stat(mFilename.c_str(), out);
if (err)
return err;
out->st_size = getLength();
return 0;
}
size_t MetadataFile::getLength() const
{
size_t totalSize = 0;
auto &objects = jsontree->get_child("objects");
if (!objects.empty())
{
auto &lastObject = objects.back().second;
totalSize = lastObject.get<off_t>("offset") + lastObject.get<size_t>("length");
}
return totalSize;
size_t totalSize = 0;
auto& objects = jsontree->get_child("objects");
if (!objects.empty())
{
auto& lastObject = objects.back().second;
totalSize = lastObject.get<off_t>("offset") + lastObject.get<size_t>("length");
}
return totalSize;
}
bool MetadataFile::exists() const
{
return _exists;
return _exists;
}
vector<metadataObject> MetadataFile::metadataRead(off_t offset, size_t length) const
{
// this version assumes mObjects is sorted by offset, and there are no gaps between objects
vector<metadataObject> ret;
size_t foundLen = 0;
/* For first cut of optimizations, going to generate mObjects as it was to use the existing alg
rather than write a new alg.
*/
set<metadataObject> mObjects;
BOOST_FOREACH(const boost::property_tree::ptree::value_type &v, jsontree->get_child("objects"))
{
mObjects.insert(metadataObject(v.second.get<uint64_t>("offset"), v.second.get<uint64_t>("length"),
v.second.get<string>("key")));
}
if (mObjects.size() == 0)
return ret;
uint64_t lastOffset = mObjects.rbegin()->offset;
auto i = mObjects.begin();
// find the first object in range
// Note, the last object in mObjects may not be full, compare the last one against its maximum
// size rather than its current size.
while (i != mObjects.end())
{
if ((uint64_t) offset <= (i->offset + i->length - 1) ||
(i->offset == lastOffset && ((uint64_t) offset <= i->offset + mpConfig->mObjectSize - 1)))
{
foundLen = (i->offset == lastOffset ? mpConfig->mObjectSize : i->length) - (offset - i->offset);
ret.push_back(*i);
++i;
break;
}
++i;
}
while (i != mObjects.end() && foundLen < length)
{
ret.push_back(*i);
foundLen += i->length;
++i;
}
// this version assumes mObjects is sorted by offset, and there are no gaps between objects
vector<metadataObject> ret;
size_t foundLen = 0;
assert(!(offset == 0 && length == getLength()) || (ret.size() == mObjects.size()));
/* For first cut of optimizations, going to generate mObjects as it was to use the existing alg
rather than write a new alg.
*/
set<metadataObject> mObjects;
BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects"))
{
mObjects.insert(metadataObject(v.second.get<uint64_t>("offset"), v.second.get<uint64_t>("length"),
v.second.get<string>("key")));
}
if (mObjects.size() == 0)
return ret;
uint64_t lastOffset = mObjects.rbegin()->offset;
auto i = mObjects.begin();
// find the first object in range
// Note, the last object in mObjects may not be full, compare the last one against its maximum
// size rather than its current size.
while (i != mObjects.end())
{
if ((uint64_t)offset <= (i->offset + i->length - 1) ||
(i->offset == lastOffset && ((uint64_t)offset <= i->offset + mpConfig->mObjectSize - 1)))
{
foundLen = (i->offset == lastOffset ? mpConfig->mObjectSize : i->length) - (offset - i->offset);
ret.push_back(*i);
++i;
break;
}
++i;
}
while (i != mObjects.end() && foundLen < length)
{
ret.push_back(*i);
foundLen += i->length;
++i;
}
assert(!(offset == 0 && length == getLength()) || (ret.size() == mObjects.size()));
return ret;
}
metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path &filename, size_t length)
metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path& filename, size_t length)
{
// this needs to handle if data write is beyond the end of the last object
// but not at start of new object
//
metadataObject addObject;
auto &objects = jsontree->get_child("objects");
if (!objects.empty())
{
auto &lastObject = objects.back().second;
addObject.offset = lastObject.get<off_t>("offset") + mpConfig->mObjectSize;
}
addObject.length = length;
addObject.key = getNewKey(filename.string(), addObject.offset, addObject.length);
boost::property_tree::ptree object;
object.put("offset", addObject.offset);
object.put("length", addObject.length);
object.put("key", addObject.key);
objects.push_back(make_pair("", object));
// this needs to handle if data write is beyond the end of the last object
// but not at start of new object
//
return addObject;
metadataObject addObject;
auto& objects = jsontree->get_child("objects");
if (!objects.empty())
{
auto& lastObject = objects.back().second;
addObject.offset = lastObject.get<off_t>("offset") + mpConfig->mObjectSize;
}
addObject.length = length;
addObject.key = getNewKey(filename.string(), addObject.offset, addObject.length);
boost::property_tree::ptree object;
object.put("offset", addObject.offset);
object.put("length", addObject.length);
object.put("key", addObject.key);
objects.push_back(make_pair("", object));
return addObject;
}
// TODO: Error handling...s
int MetadataFile::writeMetadata()
{
if (!boost::filesystem::exists(mFilename.parent_path()))
boost::filesystem::create_directories(mFilename.parent_path());
write_json(mFilename.string(), *jsontree);
_exists = true;
boost::unique_lock<boost::mutex> s(jsonCache.getMutex());
jsonCache.put(mFilename, jsontree);
if (!boost::filesystem::exists(mFilename.parent_path()))
boost::filesystem::create_directories(mFilename.parent_path());
return 0;
write_json(mFilename.string(), *jsontree);
_exists = true;
boost::unique_lock<boost::mutex> s(jsonCache.getMutex());
jsonCache.put(mFilename, jsontree);
return 0;
}
bool MetadataFile::getEntry(off_t offset, metadataObject *out) const
bool MetadataFile::getEntry(off_t offset, metadataObject* out) const
{
metadataObject addObject;
BOOST_FOREACH(const boost::property_tree::ptree::value_type &v, jsontree->get_child("objects"))
metadataObject addObject;
BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects"))
{
if (v.second.get<off_t>("offset") == offset)
{
if (v.second.get<off_t>("offset") == offset)
{
out->offset = offset;
out->length = v.second.get<size_t>("length");
out->key = v.second.get<string>("key");
return true;
}
out->offset = offset;
out->length = v.second.get<size_t>("length");
out->key = v.second.get<string>("key");
return true;
}
return false;
}
return false;
}
void MetadataFile::removeEntry(off_t offset)
{
bpt::ptree &objects = jsontree->get_child("objects");
for (bpt::ptree::iterator it = objects.begin(); it != objects.end(); ++it)
bpt::ptree& objects = jsontree->get_child("objects");
for (bpt::ptree::iterator it = objects.begin(); it != objects.end(); ++it)
{
if (it->second.get<off_t>("offset") == offset)
{
if (it->second.get<off_t>("offset") == offset)
{
objects.erase(it);
break;
}
objects.erase(it);
break;
}
}
}
void MetadataFile::removeAllEntries()
{
jsontree->get_child("objects").clear();
jsontree->get_child("objects").clear();
}
void MetadataFile::deletedMeta(const bf::path &p)
void MetadataFile::deletedMeta(const bf::path& p)
{
boost::unique_lock<boost::mutex> s(jsonCache.getMutex());
jsonCache.erase(p);
boost::unique_lock<boost::mutex> s(jsonCache.getMutex());
jsonCache.erase(p);
}
// There are more efficient ways to do it. Optimize if necessary.
void MetadataFile::breakout(const string &key, vector<string> &ret)
void MetadataFile::breakout(const string& key, vector<string>& ret)
{
int indexes[3]; // positions of each '_' delimiter
ret.clear();
indexes[0] = key.find_first_of('_');
indexes[1] = key.find_first_of('_', indexes[0] + 1);
indexes[2] = key.find_first_of('_', indexes[1] + 1);
ret.push_back(key.substr(0, indexes[0]));
ret.push_back(key.substr(indexes[0] + 1, indexes[1] - indexes[0] - 1));
ret.push_back(key.substr(indexes[1] + 1, indexes[2] - indexes[1] - 1));
ret.push_back(key.substr(indexes[2] + 1));
int indexes[3]; // positions of each '_' delimiter
ret.clear();
indexes[0] = key.find_first_of('_');
indexes[1] = key.find_first_of('_', indexes[0] + 1);
indexes[2] = key.find_first_of('_', indexes[1] + 1);
ret.push_back(key.substr(0, indexes[0]));
ret.push_back(key.substr(indexes[0] + 1, indexes[1] - indexes[0] - 1));
ret.push_back(key.substr(indexes[1] + 1, indexes[2] - indexes[1] - 1));
ret.push_back(key.substr(indexes[2] + 1));
}
string MetadataFile::getNewKeyFromOldKey(const string &key, size_t length)
string MetadataFile::getNewKeyFromOldKey(const string& key, size_t length)
{
mdfLock.lock();
boost::uuids::uuid u = boost::uuids::random_generator()();
mdfLock.unlock();
vector<string> split;
breakout(key, split);
ostringstream oss;
oss << u << "_" << split[1] << "_" << length << "_" << split[3];
return oss.str();
mdfLock.lock();
boost::uuids::uuid u = boost::uuids::random_generator()();
mdfLock.unlock();
vector<string> split;
breakout(key, split);
ostringstream oss;
oss << u << "_" << split[1] << "_" << length << "_" << split[3];
return oss.str();
}
string MetadataFile::getNewKey(string sourceName, size_t offset, size_t length)
{
mdfLock.lock();
boost::uuids::uuid u = boost::uuids::random_generator()();
mdfLock.unlock();
stringstream ss;
mdfLock.lock();
boost::uuids::uuid u = boost::uuids::random_generator()();
mdfLock.unlock();
stringstream ss;
for (uint i = 0; i < sourceName.length(); i++)
for (uint i = 0; i < sourceName.length(); i++)
{
if (sourceName[i] == '/')
{
if (sourceName[i] == '/')
{
sourceName[i] = '~';
}
sourceName[i] = '~';
}
}
ss << u << "_" << offset << "_" << length << "_" << sourceName;
return ss.str();
ss << u << "_" << offset << "_" << length << "_" << sourceName;
return ss.str();
}
off_t MetadataFile::getOffsetFromKey(const string &key)
off_t MetadataFile::getOffsetFromKey(const string& key)
{
vector<string> split;
breakout(key, split);
return stoll(split[1]);
vector<string> split;
breakout(key, split);
return stoll(split[1]);
}
string MetadataFile::getSourceFromKey(const string &key)
string MetadataFile::getSourceFromKey(const string& key)
{
vector<string> split;
breakout(key, split);
// this is to convert the munged filenames back to regular filenames
// for consistent use in IOC locks
for (uint i = 0; i < split[3].length(); i++)
if (split[3][i] == '~')
split[3][i] = '/';
return split[3];
vector<string> split;
breakout(key, split);
// this is to convert the munged filenames back to regular filenames
// for consistent use in IOC locks
for (uint i = 0; i < split[3].length(); i++)
if (split[3][i] == '~')
split[3][i] = '/';
return split[3];
}
size_t MetadataFile::getLengthFromKey(const string &key)
size_t MetadataFile::getLengthFromKey(const string& key)
{
vector<string> split;
breakout(key, split);
return stoull(split[2]);
vector<string> split;
breakout(key, split);
return stoull(split[2]);
}
// more efficient way to do these?
void MetadataFile::setOffsetInKey(string &key, off_t newOffset)
void MetadataFile::setOffsetInKey(string& key, off_t newOffset)
{
vector<string> split;
breakout(key, split);
ostringstream oss;
oss << split[0] << "_" << newOffset << "_" << split[2] << "_" << split[3];
key = oss.str();
vector<string> split;
breakout(key, split);
ostringstream oss;
oss << split[0] << "_" << newOffset << "_" << split[2] << "_" << split[3];
key = oss.str();
}
void MetadataFile::setLengthInKey(string &key, size_t newLength)
void MetadataFile::setLengthInKey(string& key, size_t newLength)
{
vector<string> split;
breakout(key, split);
ostringstream oss;
oss << split[0] << "_" << split[1] << "_" << newLength << "_" << split[3];
key = oss.str();
vector<string> split;
breakout(key, split);
ostringstream oss;
oss << split[0] << "_" << split[1] << "_" << newLength << "_" << split[3];
key = oss.str();
}
void MetadataFile::printObjects() const
{
BOOST_FOREACH(const boost::property_tree::ptree::value_type &v, jsontree->get_child("objects"))
{
printf("Name: %s Length: %zu Offset: %lld\n", v.second.get<string>("key").c_str(),
v.second.get<size_t>("length"), (long long)v.second.get<off_t>("offset"));
}
BOOST_FOREACH (const boost::property_tree::ptree::value_type& v, jsontree->get_child("objects"))
{
printf("Name: %s Length: %zu Offset: %lld\n", v.second.get<string>("key").c_str(),
v.second.get<size_t>("length"), (long long)v.second.get<off_t>("offset"));
}
}
void MetadataFile::updateEntry(off_t offset, const string &newName, size_t newLength)
void MetadataFile::updateEntry(off_t offset, const string& newName, size_t newLength)
{
for (auto &v : jsontree->get_child("objects"))
for (auto& v : jsontree->get_child("objects"))
{
if (v.second.get<off_t>("offset") == offset)
{
if (v.second.get<off_t>("offset") == offset)
{
v.second.put("key", newName);
v.second.put("length", newLength);
return;
}
v.second.put("key", newName);
v.second.put("length", newLength);
return;
}
stringstream ss;
ss << "MetadataFile::updateEntry(): failed to find object at offset " << offset;
mpLogger->log(LOG_ERR, ss.str().c_str());
throw logic_error(ss.str());
}
stringstream ss;
ss << "MetadataFile::updateEntry(): failed to find object at offset " << offset;
mpLogger->log(LOG_ERR, ss.str().c_str());
throw logic_error(ss.str());
}
void MetadataFile::updateEntryLength(off_t offset, size_t newLength)
{
for (auto &v : jsontree->get_child("objects"))
for (auto& v : jsontree->get_child("objects"))
{
if (v.second.get<off_t>("offset") == offset)
{
if (v.second.get<off_t>("offset") == offset)
{
v.second.put("length", newLength);
return;
}
v.second.put("length", newLength);
return;
}
stringstream ss;
ss << "MetadataFile::updateEntryLength(): failed to find object at offset " << offset;
mpLogger->log(LOG_ERR, ss.str().c_str());
throw logic_error(ss.str());
}
stringstream ss;
ss << "MetadataFile::updateEntryLength(): failed to find object at offset " << offset;
mpLogger->log(LOG_ERR, ss.str().c_str());
throw logic_error(ss.str());
}
off_t MetadataFile::getMetadataNewObjectOffset()
{
auto &objects = jsontree->get_child("objects");
if (objects.empty())
return 0;
auto &lastObject = jsontree->get_child("objects").back().second;
return lastObject.get<off_t>("offset") + lastObject.get<size_t>("length");
auto& objects = jsontree->get_child("objects");
if (objects.empty())
return 0;
auto& lastObject = jsontree->get_child("objects").back().second;
return lastObject.get<off_t>("offset") + lastObject.get<size_t>("length");
}
metadataObject::metadataObject() : offset(0), length(0)
{}
metadataObject::metadataObject(uint64_t _offset) : offset(_offset), length(0)
{}
metadataObject::metadataObject(uint64_t _offset, uint64_t _length, const std::string &_key) :
offset(_offset), length(_length), key(_key)
{}
MetadataFile::MetadataCache::MetadataCache() : max_lru_size(2000) // 2000 is an arbitrary #. Big enough for a large working set.
{}
inline boost::mutex & MetadataFile::MetadataCache::getMutex()
{
return mutex;
}
MetadataFile::Jsontree_t MetadataFile::MetadataCache::get(const bf::path &p)
metadataObject::metadataObject(uint64_t _offset) : offset(_offset), length(0)
{
auto it = lookup.find(p.string());
if (it != lookup.end())
{
lru.splice(lru.end(), lru, it->second.second);
return it->second.first;
}
return storagemanager::MetadataFile::Jsontree_t();
}
metadataObject::metadataObject(uint64_t _offset, uint64_t _length, const std::string& _key)
: offset(_offset), length(_length), key(_key)
{
}
MetadataFile::MetadataCache::MetadataCache()
: max_lru_size(2000) // 2000 is an arbitrary #. Big enough for a large working set.
{
}
inline boost::mutex& MetadataFile::MetadataCache::getMutex()
{
return mutex;
}
MetadataFile::Jsontree_t MetadataFile::MetadataCache::get(const bf::path& p)
{
auto it = lookup.find(p.string());
if (it != lookup.end())
{
lru.splice(lru.end(), lru, it->second.second);
return it->second.first;
}
return storagemanager::MetadataFile::Jsontree_t();
}
// note, does not change an existing jsontree. This should be OK.
void MetadataFile::MetadataCache::put(const bf::path &p, const Jsontree_t &j)
void MetadataFile::MetadataCache::put(const bf::path& p, const Jsontree_t& j)
{
string sp = p.string();
auto it = lookup.find(sp);
if (it == lookup.end())
string sp = p.string();
auto it = lookup.find(sp);
if (it == lookup.end())
{
while (lru.size() >= max_lru_size)
{
while (lru.size() >= max_lru_size)
{
lookup.erase(lru.front());
lru.pop_front();
}
lru.push_back(sp);
Lru_t::iterator last = lru.end();
lookup.emplace(sp, make_pair(j, --last));
lookup.erase(lru.front());
lru.pop_front();
}
lru.push_back(sp);
Lru_t::iterator last = lru.end();
lookup.emplace(sp, make_pair(j, --last));
}
}
void MetadataFile::MetadataCache::erase(const bf::path &p)
void MetadataFile::MetadataCache::erase(const bf::path& p)
{
auto it = lookup.find(p.string());
if (it != lookup.end())
{
lru.erase(it->second.second);
lookup.erase(it);
}
auto it = lookup.find(p.string());
if (it != lookup.end())
{
lru.erase(it->second.second);
lookup.erase(it);
}
}
MetadataFile::MetadataCache MetadataFile::jsonCache;
}
} // namespace storagemanager

212
storage-manager/src/MetadataFile.h
View File

@ -30,114 +30,118 @@
namespace storagemanager
{
struct metadataObject {
metadataObject();
metadataObject(uint64_t offset); // so we can search mObjects by integer
metadataObject(uint64_t offset, uint64_t length, const std::string &key);
uint64_t offset;
mutable uint64_t length;
mutable std::string key;
bool operator < (const metadataObject &b) const { return offset < b.offset; }
struct metadataObject
{
metadataObject();
metadataObject(uint64_t offset); // so we can search mObjects by integer
metadataObject(uint64_t offset, uint64_t length, const std::string& key);
uint64_t offset;
mutable uint64_t length;
mutable std::string key;
bool operator<(const metadataObject& b) const
{
return offset < b.offset;
}
};
class MetadataFile
{
public:
struct no_create_t {};
MetadataFile();
MetadataFile(const boost::filesystem::path &filename);
MetadataFile(const boost::filesystem::path &path, no_create_t,bool appendExt); // this one won't create it if it doesn't exist
// this ctor is 'special'. It will take an absolute path, and it will assume it points to a metafile
// meaning, that it doesn't need the metadata prefix prepended, or the .meta extension appended.
// aside from that, it will behave like the no_create ctor variant above
//MetadataFile(const boost::filesystem::path &path);
~MetadataFile();
public:
struct no_create_t
{
};
MetadataFile();
MetadataFile(const boost::filesystem::path& filename);
MetadataFile(const boost::filesystem::path& path, no_create_t,
bool appendExt); // this one won't create it if it doesn't exist
bool exists() const;
void printObjects() const;
int stat(struct stat *) const;
size_t getLength() const;
// returns the objects needed to update
std::vector<metadataObject> metadataRead(off_t offset, size_t length) const;
// updates the metadatafile with new object
//int writeMetadata(const boost::filesystem::path &filename);
int writeMetadata();
// updates the name and length fields of an entry, given the offset
void updateEntry(off_t offset, const std::string &newName, size_t newLength);
void updateEntryLength(off_t offset, size_t newLength);
metadataObject addMetadataObject(const boost::filesystem::path &filename, size_t length);
bool getEntry(off_t offset, metadataObject *out) const;
void removeEntry(off_t offset);
void removeAllEntries();
// removes p from the json cache. p should be a fully qualified metadata file
static void deletedMeta(const boost::filesystem::path &p);
static std::string getNewKeyFromOldKey(const std::string &oldKey, size_t length=0);
static std::string getNewKey(std::string sourceName, size_t offset, size_t length);
static off_t getOffsetFromKey(const std::string &key);
static std::string getSourceFromKey(const std::string &key);
static size_t getLengthFromKey(const std::string &key);
static void setOffsetInKey(std::string &key, off_t newOffset);
static void setLengthInKey(std::string &key, size_t newLength);
// breaks a key into its consitituent fields
static void breakout(const std::string &key, std::vector<std::string> &out);
off_t getMetadataNewObjectOffset();
// this will be a singleton, which stores the config used
// by all MetadataFile instances so we don't have to keep bothering Config.
// members are public b/c i don't want to write accessors right now. Also who cares.
class MetadataConfig
{
public:
static MetadataConfig *get();
size_t mObjectSize;
boost::filesystem::path msMetadataPath;
private:
MetadataConfig();
};
static void printKPIs();
typedef boost::shared_ptr<boost::property_tree::ptree> Jsontree_t;
private:
MetadataConfig *mpConfig;
SMLogging *mpLogger;
int mVersion;
int mRevision;
boost::filesystem::path mFilename;
Jsontree_t jsontree;
//std::set<metadataObject> mObjects;
bool _exists;
void makeEmptyJsonTree();
class MetadataCache
{
public:
MetadataCache();
Jsontree_t get(const boost::filesystem::path &);
void put(const boost::filesystem::path &, const Jsontree_t &);
void erase(const boost::filesystem::path &);
boost::mutex &getMutex();
private:
// there's a more efficient way to do this, KISS for now.
typedef std::list<std::string> Lru_t;
typedef std::unordered_map<std::string, std::pair<Jsontree_t, Lru_t::iterator> > Lookup_t;
Lookup_t lookup;
Lru_t lru;
uint max_lru_size;
boost::mutex mutex;
};
static MetadataCache jsonCache;
// this ctor is 'special'. It will take an absolute path, and it will assume it points to a metafile
// meaning, that it doesn't need the metadata prefix prepended, or the .meta extension appended.
// aside from that, it will behave like the no_create ctor variant above
// MetadataFile(const boost::filesystem::path &path);
~MetadataFile();
bool exists() const;
void printObjects() const;
int stat(struct stat*) const;
size_t getLength() const;
// returns the objects needed to update
std::vector<metadataObject> metadataRead(off_t offset, size_t length) const;
// updates the metadatafile with new object
// int writeMetadata(const boost::filesystem::path &filename);
int writeMetadata();
// updates the name and length fields of an entry, given the offset
void updateEntry(off_t offset, const std::string& newName, size_t newLength);
void updateEntryLength(off_t offset, size_t newLength);
metadataObject addMetadataObject(const boost::filesystem::path& filename, size_t length);
bool getEntry(off_t offset, metadataObject* out) const;
void removeEntry(off_t offset);
void removeAllEntries();
// removes p from the json cache. p should be a fully qualified metadata file
static void deletedMeta(const boost::filesystem::path& p);
static std::string getNewKeyFromOldKey(const std::string& oldKey, size_t length = 0);
static std::string getNewKey(std::string sourceName, size_t offset, size_t length);
static off_t getOffsetFromKey(const std::string& key);
static std::string getSourceFromKey(const std::string& key);
static size_t getLengthFromKey(const std::string& key);
static void setOffsetInKey(std::string& key, off_t newOffset);
static void setLengthInKey(std::string& key, size_t newLength);
// breaks a key into its consitituent fields
static void breakout(const std::string& key, std::vector<std::string>& out);
off_t getMetadataNewObjectOffset();
// this will be a singleton, which stores the config used
// by all MetadataFile instances so we don't have to keep bothering Config.
// members are public b/c i don't want to write accessors right now. Also who cares.
class MetadataConfig
{
public:
static MetadataConfig* get();
size_t mObjectSize;
boost::filesystem::path msMetadataPath;
private:
MetadataConfig();
};
static void printKPIs();
typedef boost::shared_ptr<boost::property_tree::ptree> Jsontree_t;
private:
MetadataConfig* mpConfig;
SMLogging* mpLogger;
int mVersion;
int mRevision;
boost::filesystem::path mFilename;
Jsontree_t jsontree;
// std::set<metadataObject> mObjects;
bool _exists;
void makeEmptyJsonTree();
class MetadataCache
{
public:
MetadataCache();
Jsontree_t get(const boost::filesystem::path&);
void put(const boost::filesystem::path&, const Jsontree_t&);
void erase(const boost::filesystem::path&);
boost::mutex& getMutex();
private:
// there's a more efficient way to do this, KISS for now.
typedef std::list<std::string> Lru_t;
typedef std::unordered_map<std::string, std::pair<Jsontree_t, Lru_t::iterator> > Lookup_t;
Lookup_t lookup;
Lru_t lru;
uint max_lru_size;
boost::mutex mutex;
};
static MetadataCache jsonCache;
};
}
} // namespace storagemanager

105
storage-manager/src/OpenTask.cpp
View File

@ -15,8 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "OpenTask.h"
#include "messageFormat.h"
#include "SMLogging.h"
@ -28,7 +26,6 @@ using namespace std;
namespace storagemanager
{
OpenTask::OpenTask(int sock, uint len) : PosixTask(sock, len)
{
}
@ -39,57 +36,57 @@ OpenTask::~OpenTask()
bool OpenTask::run()
{
/*
get the parameters
call IOManager to do the work
return the result
*/
SMLogging* logger = SMLogging::get();
int success;
uint8_t buf[1024] = {0};
if (getLength() > 1023)
{
handleError("OpenTask read1", ENAMETOOLONG);
return true;
}
success = read(buf, getLength());
if (success<0)
{
handleError("OpenTask read2", errno);
return false;
}
open_cmd *cmd = (open_cmd *) buf;
/*
get the parameters
call IOManager to do the work
return the result
*/
SMLogging* logger = SMLogging::get();
int success;
uint8_t buf[1024] = {0};
#ifdef SM_TRACE
logger->log(LOG_DEBUG,"open filename %s mode %o.",cmd->filename,cmd->openmode);
#endif
sm_response *resp = (sm_response *) buf;
int err;
try
{
err = ioc->open(cmd->filename, cmd->openmode, (struct stat *) &resp->payload);
}
catch (exception &e)
{
logger->log(LOG_ERR, "OpenTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err)
{
handleError("OpenTask open", errno);
return true;
}
resp->returnCode = 0;
success = write(*resp, sizeof(struct stat));
if (!success)
handleError("OpenTask write", errno);
return success;
if (getLength() > 1023)
{
handleError("OpenTask read1", ENAMETOOLONG);
return true;
}
success = read(buf, getLength());
if (success < 0)
{
handleError("OpenTask read2", errno);
return false;
}
open_cmd* cmd = (open_cmd*)buf;
#ifdef SM_TRACE
logger->log(LOG_DEBUG, "open filename %s mode %o.", cmd->filename, cmd->openmode);
#endif
sm_response* resp = (sm_response*)buf;
int err;
try
{
err = ioc->open(cmd->filename, cmd->openmode, (struct stat*)&resp->payload);
}
catch (exception& e)
{
logger->log(LOG_ERR, "OpenTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err)
{
handleError("OpenTask open", errno);
return true;
}
resp->returnCode = 0;
success = write(*resp, sizeof(struct stat));
if (!success)
handleError("OpenTask write", errno);
return success;
}
}
} // namespace storagemanager

21
storage-manager/src/OpenTask.h
View File

@ -15,25 +15,22 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "PosixTask.h"
namespace storagemanager
{
class OpenTask : public PosixTask
{
public:
OpenTask(int sock, uint length);
virtual ~OpenTask();
bool run();
private:
OpenTask();
public:
OpenTask(int sock, uint length);
virtual ~OpenTask();
bool run();
private:
OpenTask();
};
}
} // namespace storagemanager

509
storage-manager/src/Ownership.cpp
View File

@ -26,303 +26,308 @@
#include <boost/filesystem.hpp>
using namespace std;
namespace bf=boost::filesystem;
namespace bf = boost::filesystem;
namespace storagemanager
{
Ownership::Ownership()
{
Config *config = Config::get();
logger = SMLogging::get();
string sPrefixDepth = config->getValue("ObjectStorage", "common_prefix_depth");
if (sPrefixDepth.empty())
{
const char *msg = "Ownership: Need to specify ObjectStorage/common_prefix_depth in the storagemanager.cnf file";
logger->log(LOG_CRIT, msg);
throw runtime_error(msg);
}
try
{
prefixDepth = stoul(sPrefixDepth, NULL, 0);
}
catch (invalid_argument &e)
{
const char *msg = "Ownership: Invalid value in ObjectStorage/common_prefix_depth";
logger->log(LOG_CRIT, msg);
throw runtime_error(msg);
}
metadataPrefix = config->getValue("ObjectStorage", "metadata_path");
if (metadataPrefix.empty())
{
const char *msg = "Ownership: Need to specify ObjectStorage/metadata_path in the storagemanager.cnf file";
logger->log(LOG_CRIT, msg);
throw runtime_error(msg);
}
monitor = new Monitor(this);
Config* config = Config::get();
logger = SMLogging::get();
string sPrefixDepth = config->getValue("ObjectStorage", "common_prefix_depth");
if (sPrefixDepth.empty())
{
const char* msg =
"Ownership: Need to specify ObjectStorage/common_prefix_depth in the storagemanager.cnf file";
logger->log(LOG_CRIT, msg);
throw runtime_error(msg);
}
try
{
prefixDepth = stoul(sPrefixDepth, NULL, 0);
}
catch (invalid_argument& e)
{
const char* msg = "Ownership: Invalid value in ObjectStorage/common_prefix_depth";
logger->log(LOG_CRIT, msg);
throw runtime_error(msg);
}
metadataPrefix = config->getValue("ObjectStorage", "metadata_path");
if (metadataPrefix.empty())
{
const char* msg = "Ownership: Need to specify ObjectStorage/metadata_path in the storagemanager.cnf file";
logger->log(LOG_CRIT, msg);
throw runtime_error(msg);
}
monitor = new Monitor(this);
}
Ownership::~Ownership()
{
delete monitor;
for (auto &it : ownedPrefixes)
releaseOwnership(it.first, true);
delete monitor;
for (auto& it : ownedPrefixes)
releaseOwnership(it.first, true);
}
bf::path Ownership::get(const bf::path &p, bool getOwnership)
bf::path Ownership::get(const bf::path& p, bool getOwnership)
{
bf::path ret, prefix, normalizedPath(p);
bf::path::const_iterator pit;
int i, levels;
bf::path ret, prefix, normalizedPath(p);
bf::path::const_iterator pit;
int i, levels;
normalizedPath.normalize();
//cerr << "Ownership::get() param = " << normalizedPath.string() << endl;
if (prefixDepth > 0)
{
for (i = 0, pit = normalizedPath.begin(); i <= prefixDepth && pit != normalizedPath.end(); ++i, ++pit)
;
if (pit != normalizedPath.end())
prefix = *pit;
//cerr << "prefix is " << prefix.string() << endl;
for (levels = 0; pit != normalizedPath.end(); ++levels, ++pit)
ret /= *pit;
if (!getOwnership)
return ret;
if (levels <= 1)
throw runtime_error("Ownership: given path " + normalizedPath.string() +
" does not have minimum number of directories");
}
else
{
ret = normalizedPath;
prefix = *(normalizedPath.begin());
}
normalizedPath.normalize();
// cerr << "Ownership::get() param = " << normalizedPath.string() << endl;
if (prefixDepth > 0)
{
for (i = 0, pit = normalizedPath.begin(); i <= prefixDepth && pit != normalizedPath.end(); ++i, ++pit)
;
if (pit != normalizedPath.end())
prefix = *pit;
// cerr << "prefix is " << prefix.string() << endl;
for (levels = 0; pit != normalizedPath.end(); ++levels, ++pit)
ret /= *pit;
if (!getOwnership)
return ret;
mutex.lock();
if (ownedPrefixes.find(prefix) == ownedPrefixes.end())
{
mutex.unlock();
takeOwnership(prefix);
}
else
{
// todo... replace this polling, and the similar polling in Cache, with proper condition vars.
while (ownedPrefixes[prefix] == false)
{
mutex.unlock();
sleep(1);
mutex.lock();
}
mutex.unlock();
}
//cerr << "returning " << ret.string() << endl;
return ret;
if (levels <= 1)
throw runtime_error("Ownership: given path " + normalizedPath.string() +
" does not have minimum number of directories");
}
else
{
ret = normalizedPath;
prefix = *(normalizedPath.begin());
}
if (!getOwnership)
return ret;
mutex.lock();
if (ownedPrefixes.find(prefix) == ownedPrefixes.end())
{
mutex.unlock();
takeOwnership(prefix);
}
else
{
// todo... replace this polling, and the similar polling in Cache, with proper condition vars.
while (ownedPrefixes[prefix] == false)
{
mutex.unlock();
sleep(1);
mutex.lock();
}
mutex.unlock();
}
// cerr << "returning " << ret.string() << endl;
return ret;
}
// minor timesaver
#define TOUCH(p, f) { \
int fd = ::open((metadataPrefix/p/f).string().c_str(), O_TRUNC | O_CREAT | O_WRONLY, 0660); \
if (fd >= 0) \
::close(fd); \
else \
{ \
char buf[80]; int saved_errno = errno; \
cerr << "failed to touch " << metadataPrefix/p/f << " got " << strerror_r(saved_errno, buf, 80) << endl; \
} \
#define TOUCH(p, f) \
{ \
int fd = ::open((metadataPrefix / p / f).string().c_str(), O_TRUNC | O_CREAT | O_WRONLY, 0660); \
if (fd >= 0) \
::close(fd); \
else \
{ \
char buf[80]; \
int saved_errno = errno; \
cerr << "failed to touch " << metadataPrefix / p / f << " got " << strerror_r(saved_errno, buf, 80) \
<< endl; \
} \
}
#define DELETE(p, f) ::unlink((metadataPrefix / p / f).string().c_str());
void Ownership::touchFlushing(const bf::path& prefix, volatile bool* doneFlushing) const
{
while (!*doneFlushing)
{
TOUCH(prefix, "FLUSHING");
try
{
boost::this_thread::sleep_for(boost::chrono::seconds(1));
}
catch (boost::thread_interrupted&)
{
}
}
}
#define DELETE(p, f) ::unlink((metadataPrefix/p/f).string().c_str());
void Ownership::touchFlushing(const bf::path &prefix, volatile bool *doneFlushing) const
void Ownership::releaseOwnership(const bf::path& p, bool isDtor)
{
while (!*doneFlushing)
{
TOUCH(prefix, "FLUSHING");
try
{
boost::this_thread::sleep_for(boost::chrono::seconds(1));
}
catch (boost::thread_interrupted &)
{ }
}
}
logger->log(LOG_DEBUG, "Ownership: releasing ownership of %s", p.string().c_str());
boost::unique_lock<boost::mutex> s(mutex);
void Ownership::releaseOwnership(const bf::path &p, bool isDtor)
{
logger->log(LOG_DEBUG, "Ownership: releasing ownership of %s", p.string().c_str());
boost::unique_lock<boost::mutex> s(mutex);
auto it = ownedPrefixes.find(p);
if (it == ownedPrefixes.end())
{
logger->log(LOG_DEBUG, "Ownership::releaseOwnership(): told to disown %s, but do not own it", p.string().c_str());
return;
}
if (isDtor)
{
// This is a quick release. If this is being destroyed, then it is through the graceful
// shutdown mechanism, which will flush data separately.
DELETE(p, "OWNED");
DELETE(p, "FLUSHING");
return;
}
else
ownedPrefixes.erase(it);
s.unlock();
volatile bool done = false;
// start flushing
boost::thread xfer([this, &p, &done] { this->touchFlushing(p, &done); });
Synchronizer::get()->dropPrefix(p);
Cache::get()->dropPrefix(p);
done = true;
xfer.interrupt();
xfer.join();
// update state
auto it = ownedPrefixes.find(p);
if (it == ownedPrefixes.end())
{
logger->log(LOG_DEBUG, "Ownership::releaseOwnership(): told to disown %s, but do not own it",
p.string().c_str());
return;
}
if (isDtor)
{
// This is a quick release. If this is being destroyed, then it is through the graceful
// shutdown mechanism, which will flush data separately.
DELETE(p, "OWNED");
DELETE(p, "FLUSHING");
return;
}
else
ownedPrefixes.erase(it);
s.unlock();
volatile bool done = false;
// start flushing
boost::thread xfer([this, &p, &done] { this->touchFlushing(p, &done); });
Synchronizer::get()->dropPrefix(p);
Cache::get()->dropPrefix(p);
done = true;
xfer.interrupt();
xfer.join();
// update state
DELETE(p, "OWNED");
DELETE(p, "FLUSHING");
}
void Ownership::_takeOwnership(const bf::path &p)
void Ownership::_takeOwnership(const bf::path& p)
{
logger->log(LOG_DEBUG, "Ownership: taking ownership of %s", p.string().c_str());
DELETE(p, "FLUSHING");
DELETE(p, "REQUEST_TRANSFER");
// TODO: need to consider errors taking ownership
TOUCH(p, "OWNED");
mutex.lock();
ownedPrefixes[p] = true;
mutex.unlock();
Synchronizer::get()->newPrefix(p);
Cache::get()->newPrefix(p);
logger->log(LOG_DEBUG, "Ownership: taking ownership of %s", p.string().c_str());
DELETE(p, "FLUSHING");
DELETE(p, "REQUEST_TRANSFER");
// TODO: need to consider errors taking ownership
TOUCH(p, "OWNED");
mutex.lock();
ownedPrefixes[p] = true;
mutex.unlock();
Synchronizer::get()->newPrefix(p);
Cache::get()->newPrefix(p);
}
void Ownership::takeOwnership(const bf::path &p)
void Ownership::takeOwnership(const bf::path& p)
{
// If the prefix doesn't exist, ownership isn't possible yet.
if (!bf::is_directory(metadataPrefix/p))
return;
// If the prefix doesn't exist, ownership isn't possible yet.
if (!bf::is_directory(metadataPrefix / p))
return;
boost::unique_lock<boost::mutex> s(mutex);
auto it = ownedPrefixes.find(p);
if (it != ownedPrefixes.end())
return;
ownedPrefixes[p] = NULL;
s.unlock();
bool okToTransfer = false;
struct stat statbuf;
int err;
char buf[80];
bf::path ownedPath = metadataPrefix/p/"OWNED";
bf::path flushingPath = metadataPrefix/p/"FLUSHING";
// if it's not already owned, then we can take possession
err = ::stat(ownedPath.string().c_str(), &statbuf);
if (err && errno == ENOENT)
{
_takeOwnership(p);
return;
}
TOUCH(p, "REQUEST_TRANSFER");
time_t lastFlushTime = time(NULL);
while (!okToTransfer && time(NULL) < lastFlushTime + 10)
{
// if the OWNED file is deleted or if the flushing file isn't touched after 10 secs
// it is ok to take possession.
err = ::stat(ownedPath.string().c_str(), &statbuf);
if (err)
{
if (errno == ENOENT)
okToTransfer = true;
else
logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s", strerror_r(errno, buf, 80),
ownedPath.string().c_str());
}
err = ::stat(flushingPath.string().c_str(), &statbuf);
if (err && errno != ENOENT)
logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s", strerror_r(errno, buf, 80),
flushingPath.string().c_str());
else
{
logger->log(LOG_DEBUG, "Ownership: waiting to get %s", p.string().c_str());
if (!err)
lastFlushTime = statbuf.st_mtime;
}
if (!okToTransfer)
sleep(1);
}
boost::unique_lock<boost::mutex> s(mutex);
auto it = ownedPrefixes.find(p);
if (it != ownedPrefixes.end())
return;
ownedPrefixes[p] = NULL;
s.unlock();
bool okToTransfer = false;
struct stat statbuf;
int err;
char buf[80];
bf::path ownedPath = metadataPrefix / p / "OWNED";
bf::path flushingPath = metadataPrefix / p / "FLUSHING";
// if it's not already owned, then we can take possession
err = ::stat(ownedPath.string().c_str(), &statbuf);
if (err && errno == ENOENT)
{
_takeOwnership(p);
return;
}
TOUCH(p, "REQUEST_TRANSFER");
time_t lastFlushTime = time(NULL);
while (!okToTransfer && time(NULL) < lastFlushTime + 10)
{
// if the OWNED file is deleted or if the flushing file isn't touched after 10 secs
// it is ok to take possession.
err = ::stat(ownedPath.string().c_str(), &statbuf);
if (err)
{
if (errno == ENOENT)
okToTransfer = true;
else
logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s",
strerror_r(errno, buf, 80), ownedPath.string().c_str());
}
err = ::stat(flushingPath.string().c_str(), &statbuf);
if (err && errno != ENOENT)
logger->log(LOG_CRIT, "Ownership::takeOwnership(): got '%s' doing stat of %s",
strerror_r(errno, buf, 80), flushingPath.string().c_str());
else
{
logger->log(LOG_DEBUG, "Ownership: waiting to get %s", p.string().c_str());
if (!err)
lastFlushTime = statbuf.st_mtime;
}
if (!okToTransfer)
sleep(1);
}
_takeOwnership(p);
}
Ownership::Monitor::Monitor(Ownership *_owner) : owner(_owner), stop(false)
Ownership::Monitor::Monitor(Ownership* _owner) : owner(_owner), stop(false)
{
thread = boost::thread([this] { this->watchForInterlopers(); });
thread = boost::thread([this] { this->watchForInterlopers(); });
}
Ownership::Monitor::~Monitor()
{
stop = true;
thread.interrupt();
thread.join();
stop = true;
thread.interrupt();
thread.join();
}
void Ownership::Monitor::watchForInterlopers()
{
// look for requests to transfer ownership
struct stat statbuf;
int err;
char buf[80];
vector<bf::path> releaseList;
while (!stop)
// look for requests to transfer ownership
struct stat statbuf;
int err;
char buf[80];
vector<bf::path> releaseList;
while (!stop)
{
releaseList.clear();
boost::unique_lock<boost::mutex> s(owner->mutex);
for (auto& prefix : owner->ownedPrefixes)
{
releaseList.clear();
boost::unique_lock<boost::mutex> s(owner->mutex);
for (auto &prefix : owner->ownedPrefixes)
{
if (stop)
break;
if (prefix.second == false)
continue;
bf::path p(owner->metadataPrefix/(prefix.first)/"REQUEST_TRANSFER");
const char *cp = p.string().c_str();
if (stop)
break;
if (prefix.second == false)
continue;
bf::path p(owner->metadataPrefix / (prefix.first) / "REQUEST_TRANSFER");
const char* cp = p.string().c_str();
err = ::stat(cp, &statbuf);
// release it if there's a release request only. Log it if there's an error other than
// that the file isn't there.
if (err == 0)
releaseList.push_back(prefix.first);
if (err < 0 && errno != ENOENT)
owner->logger->log(LOG_ERR, "Runner::watchForInterlopers(): failed to stat %s, got %s", cp,
strerror_r(errno, buf, 80));
}
s.unlock();
for (auto &prefix : releaseList)
owner->releaseOwnership(prefix);
if (stop)
break;
try
{
boost::this_thread::sleep_for(boost::chrono::seconds(1));
}
catch (boost::thread_interrupted &)
{ }
err = ::stat(cp, &statbuf);
// release it if there's a release request only. Log it if there's an error other than
// that the file isn't there.
if (err == 0)
releaseList.push_back(prefix.first);
if (err < 0 && errno != ENOENT)
owner->logger->log(LOG_ERR, "Runner::watchForInterlopers(): failed to stat %s, got %s", cp,
strerror_r(errno, buf, 80));
}
s.unlock();
for (auto& prefix : releaseList)
owner->releaseOwnership(prefix);
if (stop)
break;
try
{
boost::this_thread::sleep_for(boost::chrono::seconds(1));
}
catch (boost::thread_interrupted&)
{
}
}
}
}
} // namespace storagemanager

78
storage-manager/src/Ownership.h
View File

@ -23,56 +23,54 @@
#include <map>
#include "SMLogging.h"
/* This class tracks the ownership of each prefix and manages ownership transfer.
/* This class tracks the ownership of each prefix and manages ownership transfer.
Could we come up with a better name btw? */
namespace storagemanager
{
class Ownership : public boost::noncopyable
{
public:
Ownership();
~Ownership();
public:
Ownership();
~Ownership();
bool sharedFS();
// returns the path "right shifted" by prefixDepth, and with ownership of that path.
// on error it throws a runtime exception
// setting getOwnership to false will return the modified path but not also take ownership
// of the returned prefix.
boost::filesystem::path get(const boost::filesystem::path &, bool getOwnership=true);
private:
int prefixDepth;
boost::filesystem::path metadataPrefix;
SMLogging *logger;
void touchFlushing(const boost::filesystem::path &, volatile bool *) const;
void takeOwnership(const boost::filesystem::path &);
void releaseOwnership(const boost::filesystem::path &, bool isDtor = false);
void _takeOwnership(const boost::filesystem::path &);
struct Monitor
{
Monitor(Ownership *);
~Monitor();
boost::thread thread;
Ownership *owner;
volatile bool stop;
void watchForInterlopers();
};
// maps a prefix to a state. ownedPrefixes[p] == false means it's being init'd, == true means it's ready for use.
std::map<boost::filesystem::path, bool> ownedPrefixes;
Monitor *monitor;
boost::mutex mutex;
bool sharedFS();
// returns the path "right shifted" by prefixDepth, and with ownership of that path.
// on error it throws a runtime exception
// setting getOwnership to false will return the modified path but not also take ownership
// of the returned prefix.
boost::filesystem::path get(const boost::filesystem::path&, bool getOwnership = true);
private:
int prefixDepth;
boost::filesystem::path metadataPrefix;
SMLogging* logger;
void touchFlushing(const boost::filesystem::path&, volatile bool*) const;
void takeOwnership(const boost::filesystem::path&);
void releaseOwnership(const boost::filesystem::path&, bool isDtor = false);
void _takeOwnership(const boost::filesystem::path&);
struct Monitor
{
Monitor(Ownership*);
~Monitor();
boost::thread thread;
Ownership* owner;
volatile bool stop;
void watchForInterlopers();
};
// maps a prefix to a state. ownedPrefixes[p] == false means it's being init'd, == true means it's ready
// for use.
std::map<boost::filesystem::path, bool> ownedPrefixes;
Monitor* monitor;
boost::mutex mutex;
};
inline bool Ownership::sharedFS()
{
return prefixDepth >= 0;
}
return prefixDepth >= 0;
}
} // namespace storagemanager

48
storage-manager/src/PingTask.cpp
View File

@ -15,14 +15,12 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "PingTask.h"
#include "messageFormat.h"
#include <errno.h>
namespace storagemanager
{
PingTask::PingTask(int sock, uint len) : PosixTask(sock, len)
{
}
@ -33,28 +31,28 @@ PingTask::~PingTask()
bool PingTask::run()
{
// not much to check on for Milestone 1
uint8_t buf;
if (getLength() > 1)
{
handleError("PingTask", E2BIG);
return true;
}
// consume the msg
int success = read(&buf, getLength());
if (success<0)
{
handleError("PingTask", errno);
return false;
}
// send generic success response
sm_response ret;
ret.returnCode = 0;
success = write(ret, 0);
return success;
// not much to check on for Milestone 1
uint8_t buf;
if (getLength() > 1)
{
handleError("PingTask", E2BIG);
return true;
}
// consume the msg
int success = read(&buf, getLength());
if (success < 0)
{
handleError("PingTask", errno);
return false;
}
// send generic success response
sm_response ret;
ret.returnCode = 0;
success = write(ret, 0);
return success;
}
}
} // namespace storagemanager

21
storage-manager/src/PingTask.h
View File

@ -15,25 +15,22 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "PosixTask.h"
namespace storagemanager
{
class PingTask : public PosixTask
{
public:
PingTask(int sock, uint length);
virtual ~PingTask();
bool run();
private:
PingTask();
public:
PingTask(int sock, uint length);
virtual ~PingTask();
bool run();
private:
PingTask();
};
}
} // namespace storagemanager

286
storage-manager/src/PosixTask.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "PosixTask.h"
#include "messageFormat.h"
#include "SMLogging.h"
@ -30,193 +29,192 @@ using namespace std;
namespace storagemanager
{
PosixTask::PosixTask(int _sock, uint _length) :
sock(_sock),
totalLength(_length),
remainingLengthInStream(_length),
remainingLengthForCaller(_length),
bufferPos(0),
bufferLen(0)
PosixTask::PosixTask(int _sock, uint _length)
: sock(_sock)
, totalLength(_length)
, remainingLengthInStream(_length)
, remainingLengthForCaller(_length)
, bufferPos(0)
, bufferLen(0)
{
ioc = IOCoordinator::get();
ioc = IOCoordinator::get();
}
PosixTask::~PosixTask()
{
assert(remainingLengthForCaller == 0);
assert(remainingLengthInStream == 0);
consumeMsg();
assert(remainingLengthForCaller == 0);
assert(remainingLengthInStream == 0);
consumeMsg();
}
void PosixTask::handleError(const char *name, int errCode)
void PosixTask::handleError(const char* name, int errCode)
{
SMLogging* logger = SMLogging::get();
char buf[80];
// send an error response if possible
sm_response *resp = (sm_response *) buf;
resp->returnCode = -1;
*((int *) resp->payload) = errCode;
bool success = write(*resp, 4);
if (!success)
logger->log(LOG_ERR, "%s caught an error: %s.", name, strerror_r(errCode, buf, 80));
SMLogging* logger = SMLogging::get();
char buf[80];
// send an error response if possible
sm_response* resp = (sm_response*)buf;
resp->returnCode = -1;
*((int*)resp->payload) = errCode;
bool success = write(*resp, 4);
if (!success)
logger->log(LOG_ERR, "%s caught an error: %s.", name, strerror_r(errCode, buf, 80));
}
uint PosixTask::getRemainingLength()
{
return remainingLengthForCaller;
return remainingLengthForCaller;
}
uint PosixTask::getLength()
{
return totalLength;
return totalLength;
}
// todo, need this to return an int instead of a bool b/c it modifies the length of the read
int PosixTask::read(uint8_t *buf, uint length)
int PosixTask::read(uint8_t* buf, uint length)
{
if (length > remainingLengthForCaller)
length = remainingLengthForCaller;
if (length == 0)
return 0;
uint count = 0;
int err;
if (length > remainingLengthForCaller)
length = remainingLengthForCaller;
if (length == 0)
return 0;
// copy data from the local buffer first.
uint dataInBuffer = bufferLen - bufferPos;
uint count = 0;
int err;
if (length <= dataInBuffer)
{
memcpy(buf, &localBuffer[bufferPos], length);
count = length;
bufferPos += length;
remainingLengthForCaller -= length;
}
else if (dataInBuffer > 0)
{
memcpy(buf, &localBuffer[bufferPos], dataInBuffer);
count = dataInBuffer;
bufferPos += dataInBuffer;
remainingLengthForCaller -= dataInBuffer;
}
// read the remaining requested amount from the stream.
// ideally, combine the recv here with the recv below that refills the local
// buffer.
while (count < length)
{
err = ::recv(sock, &buf[count], length - count, 0);
if (err < 0)
return err;
// copy data from the local buffer first.
uint dataInBuffer = bufferLen - bufferPos;
count += err;
remainingLengthInStream -= err;
remainingLengthForCaller -= err;
}
/* The caller's request has been satisfied here. If there is remaining data in the stream
get what's available. */
primeBuffer();
return count;
if (length <= dataInBuffer)
{
memcpy(buf, &localBuffer[bufferPos], length);
count = length;
bufferPos += length;
remainingLengthForCaller -= length;
}
else if (dataInBuffer > 0)
{
memcpy(buf, &localBuffer[bufferPos], dataInBuffer);
count = dataInBuffer;
bufferPos += dataInBuffer;
remainingLengthForCaller -= dataInBuffer;
}
// read the remaining requested amount from the stream.
// ideally, combine the recv here with the recv below that refills the local
// buffer.
while (count < length)
{
err = ::recv(sock, &buf[count], length - count, 0);
if (err < 0)
return err;
count += err;
remainingLengthInStream -= err;
remainingLengthForCaller -= err;
}
/* The caller's request has been satisfied here. If there is remaining data in the stream
get what's available. */
primeBuffer();
return count;
}
void PosixTask::primeBuffer()
{
if (remainingLengthInStream > 0)
if (remainingLengthInStream > 0)
{
// Reset the buffer to allow a larger read.
if (bufferLen == bufferPos)
{
// Reset the buffer to allow a larger read.
if (bufferLen == bufferPos)
{
bufferLen = 0;
bufferPos = 0;
}
else if (bufferLen - bufferPos < 1024) // if < 1024 in the buffer, move data to the front
{
// debating whether it is more efficient to use a circular buffer + more
// recv's, or to move data to reduce the # of recv's. WAG: moving data.
memmove(localBuffer, &localBuffer[bufferPos], bufferLen - bufferPos);
bufferLen -= bufferPos;
bufferPos = 0;
}
uint toRead = min(remainingLengthInStream, bufferSize - bufferLen);
int err = ::recv(sock, &localBuffer[bufferLen], toRead, MSG_DONTWAIT);
// ignoring errors here since this is supposed to be silent.
// errors will be caught by the next read
if (err > 0)
{
bufferLen += err;
remainingLengthInStream -= err;
}
bufferLen = 0;
bufferPos = 0;
}
else if (bufferLen - bufferPos < 1024) // if < 1024 in the buffer, move data to the front
{
// debating whether it is more efficient to use a circular buffer + more
// recv's, or to move data to reduce the # of recv's. WAG: moving data.
memmove(localBuffer, &localBuffer[bufferPos], bufferLen - bufferPos);
bufferLen -= bufferPos;
bufferPos = 0;
}
uint toRead = min(remainingLengthInStream, bufferSize - bufferLen);
int err = ::recv(sock, &localBuffer[bufferLen], toRead, MSG_DONTWAIT);
// ignoring errors here since this is supposed to be silent.
// errors will be caught by the next read
if (err > 0)
{
bufferLen += err;
remainingLengthInStream -= err;
}
}
}
bool PosixTask::write(const uint8_t *buf, uint len)
bool PosixTask::write(const uint8_t* buf, uint len)
{
int err;
uint count = 0;
while (count < len)
{
err = ::send(sock, &buf[count], len - count, 0);
if (err < 0)
return false;
count += err;
}
return true;
int err;
uint count = 0;
while (count < len)
{
err = ::send(sock, &buf[count], len - count, 0);
if (err < 0)
return false;
count += err;
}
return true;
}
bool PosixTask::write(sm_response &resp, uint payloadLength)
bool PosixTask::write(sm_response& resp, uint payloadLength)
{
int err;
uint count = 0;
uint8_t *buf = (uint8_t *) &resp;
resp.header.type = SM_MSG_START;
resp.header.flags = 0;
resp.header.payloadLen = payloadLength + sizeof(sm_response) - sizeof(sm_msg_header);
uint toSend = payloadLength + sizeof(sm_response);
while (count < toSend)
{
err = ::send(sock, &buf[count], toSend - count, 0);
if (err < 0)
return false;
count += err;
}
return true;
int err;
uint count = 0;
uint8_t* buf = (uint8_t*)&resp;
resp.header.type = SM_MSG_START;
resp.header.flags = 0;
resp.header.payloadLen = payloadLength + sizeof(sm_response) - sizeof(sm_msg_header);
uint toSend = payloadLength + sizeof(sm_response);
while (count < toSend)
{
err = ::send(sock, &buf[count], toSend - count, 0);
if (err < 0)
return false;
count += err;
}
return true;
}
bool PosixTask::write(const vector<uint8_t> &buf)
bool PosixTask::write(const vector<uint8_t>& buf)
{
return write(&buf[0], buf.size());
return write(&buf[0], buf.size());
}
void PosixTask::consumeMsg()
{
SMLogging* logger = SMLogging::get();
SMLogging* logger = SMLogging::get();
uint8_t buf[1024];
int err;
bufferLen = 0;
bufferPos = 0;
remainingLengthForCaller = 0;
while (remainingLengthInStream > 0)
uint8_t buf[1024];
int err;
bufferLen = 0;
bufferPos = 0;
remainingLengthForCaller = 0;
while (remainingLengthInStream > 0)
{
logger->log(LOG_WARNING, "PosixTask::consumeMsg(): Discarding the tail end of a partial msg.");
err = ::recv(sock, buf, min(remainingLengthInStream, 1024), 0);
if (err <= 0)
{
logger->log(LOG_WARNING, "PosixTask::consumeMsg(): Discarding the tail end of a partial msg.");
err = ::recv(sock, buf, min(remainingLengthInStream, 1024), 0);
if (err <= 0) {
remainingLengthInStream = 0;
break;
}
remainingLengthInStream -= err;
remainingLengthInStream = 0;
break;
}
remainingLengthInStream -= err;
}
}
}
} // namespace storagemanager

67
storage-manager/src/PosixTask.h
View File

@ -15,8 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <vector>
@ -28,40 +26,39 @@
namespace storagemanager
{
class PosixTask
{
public:
PosixTask(int sock, uint length);
virtual ~PosixTask();
// this should return false if there was a network error, true otherwise including for other errors
virtual bool run() = 0;
void primeBuffer();
protected:
int read(uint8_t *buf, uint length);
bool write(const std::vector<uint8_t> &buf);
bool write(sm_response &resp, uint payloadLength);
bool write(const uint8_t *buf, uint length);
void consumeMsg(); // drains the remaining portion of the message
uint getLength(); // returns the total length of the msg
uint getRemainingLength(); // returns the remaining length from the caller's perspective
void handleError(const char *name, int errCode);
IOCoordinator *ioc;
private:
PosixTask();
int sock;
int totalLength;
uint remainingLengthInStream;
uint remainingLengthForCaller;
static const uint bufferSize = 4096;
uint8_t localBuffer[bufferSize];
uint bufferPos;
uint bufferLen;
public:
PosixTask(int sock, uint length);
virtual ~PosixTask();
// this should return false if there was a network error, true otherwise including for other errors
virtual bool run() = 0;
void primeBuffer();
protected:
int read(uint8_t* buf, uint length);
bool write(const std::vector<uint8_t>& buf);
bool write(sm_response& resp, uint payloadLength);
bool write(const uint8_t* buf, uint length);
void consumeMsg(); // drains the remaining portion of the message
uint getLength(); // returns the total length of the msg
uint getRemainingLength(); // returns the remaining length from the caller's perspective
void handleError(const char* name, int errCode);
IOCoordinator* ioc;
private:
PosixTask();
int sock;
int totalLength;
uint remainingLengthInStream;
uint remainingLengthForCaller;
static const uint bufferSize = 4096;
uint8_t localBuffer[bufferSize];
uint bufferPos;
uint bufferLen;
};
}
} // namespace storagemanager

1026
storage-manager/src/PrefixCache.cpp
View File

File diff suppressed because it is too large Load Diff

243
storage-manager/src/PrefixCache.h
View File

@ -17,12 +17,11 @@
#pragma once
/* PrefixCache manages the cache for one prefix managed by SM.
/* PrefixCache manages the cache for one prefix managed by SM.
Cache is a map of prefix -> PrefixCache, and holds the items
that should be centralized like the Downloader
*/
#include "Downloader.h"
#include "SMLogging.h"
#include "Replicator.h"
@ -37,132 +36,130 @@
namespace storagemanager
{
class PrefixCache : public boost::noncopyable
{
public:
PrefixCache(const boost::filesystem::path &prefix);
virtual ~PrefixCache();
//reading fcns
// read() marks objects to be read s.t. they do not get flushed.
// after reading them, unlock the 'logical file', and call doneReading().
void read(const std::vector<std::string> &keys);
void doneReading(const std::vector<std::string> &keys);
bool exists(const std::string &key) const;
void exists(const std::vector<std::string> &keys, std::vector<bool> *out) const;
// writing fcns
// new*() fcns tell the PrefixCache data was added. After writing a set of objects,
// unlock the 'logical file', and call doneWriting().
void newObject(const std::string &key, size_t size);
void newJournalEntry(size_t size);
void doneWriting();
void deletedObject(const std::string &key, size_t size);
void deletedJournal(size_t size);
// an 'atomic' existence check & delete. Covers the object and journal. Does not delete the files.
// returns 0 if it didn't exist, 1 if the object exists, 2 if the journal exists, and 3 (1 | 2) if both exist
// This should be called while holding the file lock for key because it touches the journal file.
int ifExistsThenDelete(const std::string &key);
// rename is used when an old obj gets merged with its journal file
// the size will change in that process; sizediff is by how much
void rename(const std::string &oldKey, const std::string &newKey, ssize_t sizediff);
void setMaxCacheSize(size_t size);
void makeSpace(size_t size);
size_t getCurrentCacheSize() const;
size_t getCurrentCacheElementCount() const;
size_t getMaxCacheSize() const;
void shutdown();
public:
PrefixCache(const boost::filesystem::path& prefix);
virtual ~PrefixCache();
// test helpers
const boost::filesystem::path &getCachePath();
const boost::filesystem::path &getJournalPath();
// this will delete everything in the PrefixCache and journal paths, and empty all PrefixCache structures.
void reset();
void validateCacheSize();
private:
PrefixCache();
boost::filesystem::path cachePrefix;
boost::filesystem::path journalPrefix;
boost::filesystem::path firstDir;
size_t maxCacheSize;
size_t objectSize;
size_t currentCacheSize;
Replicator *replicator;
SMLogging *logger;
Downloader *downloader;
void populate();
void _makeSpace(size_t size);
// reading fcns
// read() marks objects to be read s.t. they do not get flushed.
// after reading them, unlock the 'logical file', and call doneReading().
void read(const std::vector<std::string>& keys);
void doneReading(const std::vector<std::string>& keys);
bool exists(const std::string& key) const;
void exists(const std::vector<std::string>& keys, std::vector<bool>* out) const;
/* The main PrefixCache structures */
// lru owns the string memory for the filenames it manages. m_lru and DNE point to those strings.
typedef std::list<std::string> LRU_t;
LRU_t lru;
struct M_LRU_element_t
{
M_LRU_element_t(const std::string &);
M_LRU_element_t(const std::string *);
M_LRU_element_t(const LRU_t::iterator &);
const std::string *key;
LRU_t::iterator lit;
};
struct KeyHasher
{
size_t operator()(const M_LRU_element_t &l) const;
};
// writing fcns
// new*() fcns tell the PrefixCache data was added. After writing a set of objects,
// unlock the 'logical file', and call doneWriting().
void newObject(const std::string& key, size_t size);
void newJournalEntry(size_t size);
void doneWriting();
void deletedObject(const std::string& key, size_t size);
void deletedJournal(size_t size);
struct KeyEquals
{
bool operator()(const M_LRU_element_t &l1, const M_LRU_element_t &l2) const;
};
typedef std::unordered_set<M_LRU_element_t, KeyHasher, KeyEquals> M_LRU_t;
M_LRU_t m_lru; // the LRU entries as a hash table
/* The do-not-evict list stuff. */
struct DNEElement
{
DNEElement(const LRU_t::iterator &);
DNEElement(const std::string &);
LRU_t::iterator key;
std::string sKey;
uint refCount;
};
struct DNEHasher
{
size_t operator()(const DNEElement &d) const;
};
struct DNEEquals
{
bool operator()(const DNEElement &d1, const DNEElement &d2) const;
};
typedef std::unordered_set<DNEElement, DNEHasher, DNEEquals> DNE_t;
DNE_t doNotEvict;
void addToDNE(const DNEElement &);
void removeFromDNE(const DNEElement &);
// the to-be-deleted set. Elements removed from the LRU but not yet deleted will be here.
// Elements are inserted and removed by makeSpace(). If read() references a file that is in this,
// it will remove it, signalling to makeSpace that it should not be deleted
struct TBDLess
{
bool operator()(const LRU_t::iterator &, const LRU_t::iterator &) const;
};
// an 'atomic' existence check & delete. Covers the object and journal. Does not delete the files.
// returns 0 if it didn't exist, 1 if the object exists, 2 if the journal exists, and 3 (1 | 2) if both
// exist This should be called while holding the file lock for key because it touches the journal file.
int ifExistsThenDelete(const std::string& key);
typedef std::set<LRU_t::iterator, TBDLess> TBD_t;
TBD_t toBeDeleted;
mutable boost::mutex lru_mutex; // protects the main PrefixCache structures & the do-not-evict set
// rename is used when an old obj gets merged with its journal file
// the size will change in that process; sizediff is by how much
void rename(const std::string& oldKey, const std::string& newKey, ssize_t sizediff);
void setMaxCacheSize(size_t size);
void makeSpace(size_t size);
size_t getCurrentCacheSize() const;
size_t getCurrentCacheElementCount() const;
size_t getMaxCacheSize() const;
void shutdown();
// test helpers
const boost::filesystem::path& getCachePath();
const boost::filesystem::path& getJournalPath();
// this will delete everything in the PrefixCache and journal paths, and empty all PrefixCache structures.
void reset();
void validateCacheSize();
private:
PrefixCache();
boost::filesystem::path cachePrefix;
boost::filesystem::path journalPrefix;
boost::filesystem::path firstDir;
size_t maxCacheSize;
size_t objectSize;
size_t currentCacheSize;
Replicator* replicator;
SMLogging* logger;
Downloader* downloader;
void populate();
void _makeSpace(size_t size);
/* The main PrefixCache structures */
// lru owns the string memory for the filenames it manages. m_lru and DNE point to those strings.
typedef std::list<std::string> LRU_t;
LRU_t lru;
struct M_LRU_element_t
{
M_LRU_element_t(const std::string&);
M_LRU_element_t(const std::string*);
M_LRU_element_t(const LRU_t::iterator&);
const std::string* key;
LRU_t::iterator lit;
};
struct KeyHasher
{
size_t operator()(const M_LRU_element_t& l) const;
};
struct KeyEquals
{
bool operator()(const M_LRU_element_t& l1, const M_LRU_element_t& l2) const;
};
typedef std::unordered_set<M_LRU_element_t, KeyHasher, KeyEquals> M_LRU_t;
M_LRU_t m_lru; // the LRU entries as a hash table
/* The do-not-evict list stuff. */
struct DNEElement
{
DNEElement(const LRU_t::iterator&);
DNEElement(const std::string&);
LRU_t::iterator key;
std::string sKey;
uint refCount;
};
struct DNEHasher
{
size_t operator()(const DNEElement& d) const;
};
struct DNEEquals
{
bool operator()(const DNEElement& d1, const DNEElement& d2) const;
};
typedef std::unordered_set<DNEElement, DNEHasher, DNEEquals> DNE_t;
DNE_t doNotEvict;
void addToDNE(const DNEElement&);
void removeFromDNE(const DNEElement&);
// the to-be-deleted set. Elements removed from the LRU but not yet deleted will be here.
// Elements are inserted and removed by makeSpace(). If read() references a file that is in this,
// it will remove it, signalling to makeSpace that it should not be deleted
struct TBDLess
{
bool operator()(const LRU_t::iterator&, const LRU_t::iterator&) const;
};
typedef std::set<LRU_t::iterator, TBDLess> TBD_t;
TBD_t toBeDeleted;
mutable boost::mutex lru_mutex; // protects the main PrefixCache structures & the do-not-evict set
};
}
} // namespace storagemanager

133
storage-manager/src/ProcessTask.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "ProcessTask.h"
#include <vector>
#include <iostream>
@ -41,97 +40,73 @@ using namespace std;
namespace storagemanager
{
ProcessTask::ProcessTask(int _sock, uint _length) : sock(_sock), length(_length), returnedSock(false)
{
assert(length > 0);
assert(length > 0);
}
ProcessTask::~ProcessTask()
{
if (!returnedSock)
(SessionManager::get())->returnSocket(sock);
if (!returnedSock)
(SessionManager::get())->returnSocket(sock);
}
void ProcessTask::handleError(int saved_errno)
{
SMLogging* logger = SMLogging::get();
SessionManager::get()->socketError(sock);
returnedSock = true;
char buf[80];
logger->log(LOG_ERR,"ProcessTask: got an error during a socket read: %s.",strerror_r(saved_errno, buf, 80));
SMLogging* logger = SMLogging::get();
SessionManager::get()->socketError(sock);
returnedSock = true;
char buf[80];
logger->log(LOG_ERR, "ProcessTask: got an error during a socket read: %s.",
strerror_r(saved_errno, buf, 80));
}
void ProcessTask::operator()()
{
/*
Read the command from the socket
Create the appropriate PosixTask
Run it
*/
vector<uint8_t> msg;
int err;
uint8_t opcode;
err = ::recv(sock, &opcode, 1, MSG_PEEK);
if (err <= 0)
{
handleError(errno);
return;
}
boost::scoped_ptr<PosixTask> task;
switch(opcode)
{
case OPEN:
task.reset(new OpenTask(sock, length));
break;
case READ:
task.reset(new ReadTask(sock, length));
break;
case WRITE:
task.reset(new WriteTask(sock, length));
break;
case STAT:
task.reset(new StatTask(sock, length));
break;
case UNLINK:
task.reset(new UnlinkTask(sock, length));
break;
case APPEND:
task.reset(new AppendTask(sock, length));
break;
case TRUNCATE:
task.reset(new TruncateTask(sock, length));
break;
case LIST_DIRECTORY:
task.reset(new ListDirectoryTask(sock, length));
break;
case PING:
task.reset(new PingTask(sock, length));
break;
case SYNC:
task.reset(new SyncTask(sock, length));
break;
case COPY:
task.reset(new CopyTask(sock, length));
break;
default:
throw runtime_error("ProcessTask: got an unknown opcode");
}
task->primeBuffer();
bool success = task->run();
if (!success)
{
SessionManager::get()->socketError(sock);
returnedSock = true;
}
else
{
SessionManager::get()->returnSocket(sock);
returnedSock = true;
}
/*
Read the command from the socket
Create the appropriate PosixTask
Run it
*/
vector<uint8_t> msg;
int err;
uint8_t opcode;
err = ::recv(sock, &opcode, 1, MSG_PEEK);
if (err <= 0)
{
handleError(errno);
return;
}
boost::scoped_ptr<PosixTask> task;
switch (opcode)
{
case OPEN: task.reset(new OpenTask(sock, length)); break;
case READ: task.reset(new ReadTask(sock, length)); break;
case WRITE: task.reset(new WriteTask(sock, length)); break;
case STAT: task.reset(new StatTask(sock, length)); break;
case UNLINK: task.reset(new UnlinkTask(sock, length)); break;
case APPEND: task.reset(new AppendTask(sock, length)); break;
case TRUNCATE: task.reset(new TruncateTask(sock, length)); break;
case LIST_DIRECTORY: task.reset(new ListDirectoryTask(sock, length)); break;
case PING: task.reset(new PingTask(sock, length)); break;
case SYNC: task.reset(new SyncTask(sock, length)); break;
case COPY: task.reset(new CopyTask(sock, length)); break;
default: throw runtime_error("ProcessTask: got an unknown opcode");
}
task->primeBuffer();
bool success = task->run();
if (!success)
{
SessionManager::get()->socketError(sock);
returnedSock = true;
}
else
{
SessionManager::get()->returnSocket(sock);
returnedSock = true;
}
}
}
} // namespace storagemanager

37
storage-manager/src/ProcessTask.h
View File

@ -15,36 +15,27 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "ThreadPool.h"
namespace storagemanager
{
class ProcessTask : public ThreadPool::Job
{
public:
ProcessTask(int sock, uint length); // _sock is the socket to read from
virtual ~ProcessTask();
void operator()();
private:
ProcessTask();
void handleError(int errCode);
int sock;
uint length;
bool returnedSock;
public:
ProcessTask(int sock, uint length); // _sock is the socket to read from
virtual ~ProcessTask();
void operator()();
private:
ProcessTask();
void handleError(int errCode);
int sock;
uint length;
bool returnedSock;
};
}
} // namespace storagemanager

115
storage-manager/src/RWLock.cpp
View File

@ -19,95 +19,94 @@
namespace storagemanager
{
RWLock::RWLock() : readersWaiting(0), readersRunning(0), writersWaiting(0), writersRunning(0)
{
}
RWLock::~RWLock()
{
assert(!readersWaiting);
assert(!readersRunning);
assert(!writersWaiting);
assert(!writersRunning);
assert(!readersWaiting);
assert(!readersRunning);
assert(!writersWaiting);
assert(!writersRunning);
}
bool RWLock::inUse()
{
boost::unique_lock<boost::mutex> s(m);
return readersWaiting || readersRunning || writersWaiting || writersRunning;
boost::unique_lock<boost::mutex> s(m);
return readersWaiting || readersRunning || writersWaiting || writersRunning;
}
void RWLock::readLock()
{
boost::unique_lock<boost::mutex> s(m);
++readersWaiting;
while (writersWaiting != 0 || writersRunning != 0)
okToRead.wait(s);
++readersRunning;
--readersWaiting;
boost::unique_lock<boost::mutex> s(m);
++readersWaiting;
while (writersWaiting != 0 || writersRunning != 0)
okToRead.wait(s);
++readersRunning;
--readersWaiting;
}
void RWLock::readLock(boost::unique_lock<boost::mutex> &l)
void RWLock::readLock(boost::unique_lock<boost::mutex>& l)
{
boost::unique_lock<boost::mutex> s(m);
l.unlock();
++readersWaiting;
while (writersWaiting != 0 || writersRunning != 0)
okToRead.wait(s);
++readersRunning;
--readersWaiting;
boost::unique_lock<boost::mutex> s(m);
l.unlock();
++readersWaiting;
while (writersWaiting != 0 || writersRunning != 0)
okToRead.wait(s);
++readersRunning;
--readersWaiting;
}
void RWLock::readUnlock()
{
boost::unique_lock<boost::mutex> s(m);
assert(readersRunning > 0);
--readersRunning;
if (readersRunning == 0 && writersWaiting != 0)
okToWrite.notify_one();
boost::unique_lock<boost::mutex> s(m);
assert(readersRunning > 0);
--readersRunning;
if (readersRunning == 0 && writersWaiting != 0)
okToWrite.notify_one();
}
void RWLock::writeLock()
{
boost::unique_lock<boost::mutex> s(m);
++writersWaiting;
while (readersRunning != 0 || writersRunning != 0)
okToWrite.wait(s);
++writersRunning;
--writersWaiting;
boost::unique_lock<boost::mutex> s(m);
++writersWaiting;
while (readersRunning != 0 || writersRunning != 0)
okToWrite.wait(s);
++writersRunning;
--writersWaiting;
}
void RWLock::writeLock(boost::unique_lock<boost::mutex> &l)
void RWLock::writeLock(boost::unique_lock<boost::mutex>& l)
{
boost::unique_lock<boost::mutex> s(m);
l.unlock();
++writersWaiting;
while (readersRunning != 0 || writersRunning != 0)
okToWrite.wait(s);
++writersRunning;
--writersWaiting;
boost::unique_lock<boost::mutex> s(m);
l.unlock();
++writersWaiting;
while (readersRunning != 0 || writersRunning != 0)
okToWrite.wait(s);
++writersRunning;
--writersWaiting;
}
void RWLock::writeUnlock()
{
boost::unique_lock<boost::mutex> s(m);
assert(writersRunning > 0);
--writersRunning;
if (writersWaiting != 0)
okToWrite.notify_one();
else if (readersWaiting != 0)
okToRead.notify_all();
boost::unique_lock<boost::mutex> s(m);
assert(writersRunning > 0);
--writersRunning;
if (writersWaiting != 0)
okToWrite.notify_one();
else if (readersWaiting != 0)
okToRead.notify_all();
}
}
} // namespace storagemanager

55
storage-manager/src/RWLock.h
View File

@ -21,32 +21,31 @@
/* Quicky impl of a read-write lock that prioritizes writers. */
namespace storagemanager
{
class RWLock
{
public:
RWLock();
~RWLock();
void readLock();
// this version will release the lock in the parameter after locking this instance
void readLock(boost::unique_lock<boost::mutex> &);
void readUnlock();
void writeLock();
// this version will release the lock in the parameter after locking this instance
void writeLock(boost::unique_lock<boost::mutex> &);
void writeUnlock();
// returns true if anything is blocked on or owns this lock instance.
bool inUse();
private:
uint readersWaiting;
uint readersRunning;
uint writersWaiting;
uint writersRunning;
boost::mutex m;
boost::condition okToWrite;
boost::condition okToRead;
};
}
class RWLock
{
public:
RWLock();
~RWLock();
void readLock();
// this version will release the lock in the parameter after locking this instance
void readLock(boost::unique_lock<boost::mutex>&);
void readUnlock();
void writeLock();
// this version will release the lock in the parameter after locking this instance
void writeLock(boost::unique_lock<boost::mutex>&);
void writeUnlock();
// returns true if anything is blocked on or owns this lock instance.
bool inUse();
private:
uint readersWaiting;
uint readersRunning;
uint writersWaiting;
uint writersRunning;
boost::mutex m;
boost::condition okToWrite;
boost::condition okToRead;
};
} // namespace storagemanager

133
storage-manager/src/ReadTask.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "ReadTask.h"
#include "messageFormat.h"
#include "SMLogging.h"
@ -25,7 +24,6 @@ using namespace std;
namespace storagemanager
{
ReadTask::ReadTask(int sock, uint len) : PosixTask(sock, len)
{
}
@ -35,77 +33,78 @@ ReadTask::~ReadTask()
}
#define check_error(msg, ret) \
if (success<0) \
{ \
handleError(msg, errno); \
return ret; \
}
if (success < 0) \
{ \
handleError(msg, errno); \
return ret; \
}
#define max(x, y) (x > y ? x : y)
bool ReadTask::run()
{
SMLogging* logger = SMLogging::get();
uint8_t buf[1024] = {0};
SMLogging* logger = SMLogging::get();
uint8_t buf[1024] = {0};
// get the parameters
if (getLength() > 1023) {
handleError("ReadTask read", EFAULT);
return true;
}
int success;
success = read(buf, getLength());
check_error("ReadTask read cmd", false);
read_cmd *cmd = (read_cmd *) buf;
#ifdef SM_TRACE
logger->log(LOG_DEBUG,"read %s count %i offset %i.",cmd->filename,cmd->count,cmd->offset);
#endif
// read from IOC, write to the socket
vector<uint8_t> outbuf;
if (cmd->count > (100 << 20))
cmd->count = (100 << 20); // cap a read request at 100MB
outbuf.resize(max(cmd->count, 4) + sizeof(sm_response));
sm_response *resp = (sm_response *) &outbuf[0];
resp->returnCode = 0;
uint payloadLen = 0;
// todo: do the reading and writing in chunks
// todo: need to make this use O_DIRECT on the IOC side
ssize_t err;
while ((uint) resp->returnCode < cmd->count)
// get the parameters
if (getLength() > 1023)
{
handleError("ReadTask read", EFAULT);
return true;
}
int success;
success = read(buf, getLength());
check_error("ReadTask read cmd", false);
read_cmd* cmd = (read_cmd*)buf;
#ifdef SM_TRACE
logger->log(LOG_DEBUG, "read %s count %i offset %i.", cmd->filename, cmd->count, cmd->offset);
#endif
// read from IOC, write to the socket
vector<uint8_t> outbuf;
if (cmd->count > (100 << 20))
cmd->count = (100 << 20); // cap a read request at 100MB
outbuf.resize(max(cmd->count, 4) + sizeof(sm_response));
sm_response* resp = (sm_response*)&outbuf[0];
resp->returnCode = 0;
uint payloadLen = 0;
// todo: do the reading and writing in chunks
// todo: need to make this use O_DIRECT on the IOC side
ssize_t err;
while ((uint)resp->returnCode < cmd->count)
{
try
{
try
{
err = ioc->read(cmd->filename, &resp->payload[resp->returnCode], cmd->offset + resp->returnCode,
cmd->count - resp->returnCode);
}
catch (exception &e)
{
logger->log(LOG_ERR, "ReadTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err < 0) {
if (resp->returnCode == 0) {
resp->returnCode = err;
payloadLen = 4;
*((int32_t *) resp->payload) = errno;
}
break;
}
if (err == 0)
break;
resp->returnCode += err;
err = ioc->read(cmd->filename, &resp->payload[resp->returnCode], cmd->offset + resp->returnCode,
cmd->count - resp->returnCode);
}
if (resp->returnCode >= 0)
payloadLen = resp->returnCode;
return write(*resp, payloadLen);
catch (exception& e)
{
logger->log(LOG_ERR, "ReadTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err < 0)
{
if (resp->returnCode == 0)
{
resp->returnCode = err;
payloadLen = 4;
*((int32_t*)resp->payload) = errno;
}
break;
}
if (err == 0)
break;
resp->returnCode += err;
}
if (resp->returnCode >= 0)
payloadLen = resp->returnCode;
return write(*resp, payloadLen);
}
}
} // namespace storagemanager

17
storage-manager/src/ReadTask.h
View File

@ -15,25 +15,22 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "PosixTask.h"
namespace storagemanager
{
class ReadTask : public PosixTask
{
public:
ReadTask(int sock, uint length);
virtual ~ReadTask();
public:
ReadTask(int sock, uint length);
virtual ~ReadTask();
bool run();
bool run();
private:
ReadTask();
private:
ReadTask();
};
}
} // namespace storagemanager

808
storage-manager/src/Replicator.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "Replicator.h"
#include "IOCoordinator.h"
#include "SMLogging.h"
@ -38,439 +37,456 @@ using namespace std;
namespace
{
storagemanager::Replicator *rep = NULL;
boost::mutex m;
}
storagemanager::Replicator* rep = NULL;
boost::mutex m;
} // namespace
namespace storagemanager
{
Replicator::Replicator()
{
mpConfig = Config::get();
mpLogger = SMLogging::get();
try
mpConfig = Config::get();
mpLogger = SMLogging::get();
try
{
msJournalPath = mpConfig->getValue("ObjectStorage", "journal_path");
if (msJournalPath.empty())
{
msJournalPath = mpConfig->getValue("ObjectStorage", "journal_path");
if (msJournalPath.empty())
{
mpLogger->log(LOG_CRIT, "ObjectStorage/journal_path is not set");
throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file");
}
mpLogger->log(LOG_CRIT, "ObjectStorage/journal_path is not set");
throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file");
}
catch (...)
{
mpLogger->log(LOG_CRIT, "Could not load metadata_path from storagemanger.cnf file.");
throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file");
}
try
{
boost::filesystem::create_directories(msJournalPath);
}
catch (exception &e)
{
syslog(LOG_CRIT, "Failed to create %s, got: %s", msJournalPath.c_str(), e.what());
throw e;
}
msCachePath = mpConfig->getValue("Cache", "path");
if (msCachePath.empty())
{
mpLogger->log(LOG_CRIT, "Cache/path is not set");
throw runtime_error("Please set Cache/path in the storagemanager.cnf file");
}
try
{
boost::filesystem::create_directories(msCachePath);
}
catch (exception &e)
{
mpLogger->log(LOG_CRIT, "Failed to create %s, got: %s", msCachePath.c_str(), e.what());
throw e;
}
repUserDataWritten = repHeaderDataWritten = replicatorObjectsCreated = replicatorJournalsCreated = 0;
}
catch (...)
{
mpLogger->log(LOG_CRIT, "Could not load metadata_path from storagemanger.cnf file.");
throw runtime_error("Please set ObjectStorage/metadata_path in the storagemanager.cnf file");
}
try
{
boost::filesystem::create_directories(msJournalPath);
}
catch (exception& e)
{
syslog(LOG_CRIT, "Failed to create %s, got: %s", msJournalPath.c_str(), e.what());
throw e;
}
msCachePath = mpConfig->getValue("Cache", "path");
if (msCachePath.empty())
{
mpLogger->log(LOG_CRIT, "Cache/path is not set");
throw runtime_error("Please set Cache/path in the storagemanager.cnf file");
}
try
{
boost::filesystem::create_directories(msCachePath);
}
catch (exception& e)
{
mpLogger->log(LOG_CRIT, "Failed to create %s, got: %s", msCachePath.c_str(), e.what());
throw e;
}
repUserDataWritten = repHeaderDataWritten = replicatorObjectsCreated = replicatorJournalsCreated = 0;
}
Replicator::~Replicator()
{
}
Replicator * Replicator::get()
Replicator* Replicator::get()
{
if (rep)
return rep;
boost::mutex::scoped_lock s(m);
if (rep)
return rep;
rep = new Replicator();
if (rep)
return rep;
boost::mutex::scoped_lock s(m);
if (rep)
return rep;
rep = new Replicator();
return rep;
}
void Replicator::printKPIs() const
{
cout << "Replicator" << endl;
cout << "\treplicatorUserDataWritten = " << repUserDataWritten << endl;
cout << "\treplicatorHeaderDataWritten = " << repHeaderDataWritten << endl;
cout << "Replicator" << endl;
cout << "\treplicatorUserDataWritten = " << repUserDataWritten << endl;
cout << "\treplicatorHeaderDataWritten = " << repHeaderDataWritten << endl;
cout << "\treplicatorObjectsCreated = " << replicatorObjectsCreated << endl;
cout << "\treplicatorJournalsCreated = " << replicatorJournalsCreated << endl;
cout << "\treplicatorObjectsCreated = " << replicatorObjectsCreated << endl;
cout << "\treplicatorJournalsCreated = " << replicatorJournalsCreated << endl;
}
#define OPEN(name, mode) \
fd = ::open(name, mode, 0600); \
if (fd < 0) \
return fd; \
ScopedCloser sc(fd);
#define OPEN(name, mode) \
fd = ::open(name, mode, 0600); \
if (fd < 0) \
return fd; \
ScopedCloser sc(fd);
int Replicator::newObject(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length )
int Replicator::newObject(const boost::filesystem::path& filename, const uint8_t* data, off_t offset,
size_t length)
{
int fd, err;
string objectFilename = msCachePath + "/" + filename.string();
int fd, err;
string objectFilename = msCachePath + "/" + filename.string();
OPEN(objectFilename.c_str(), O_WRONLY | O_CREAT);
size_t count = 0;
while (count < length) {
err = ::pwrite(fd, &data[count], length - count, offset + count);
if (err <= 0)
{
if (count > 0) // return what was successfully written
return count;
else
return err;
}
count += err;
}
repUserDataWritten += count;
++replicatorObjectsCreated;
return count;
}
int Replicator::newNullObject(const boost::filesystem::path &filename,size_t length )
{
int fd, err;
string objectFilename = msCachePath + "/" + filename.string();
OPEN(objectFilename.c_str(), O_WRONLY | O_CREAT);
err = ftruncate(fd,length);
return err;
}
ssize_t Replicator::_pwrite(int fd, const void *data, size_t length, off_t offset)
{
ssize_t err;
size_t count = 0;
uint8_t *bData = (uint8_t *) data;
do
OPEN(objectFilename.c_str(), O_WRONLY | O_CREAT);
size_t count = 0;
while (count < length)
{
err = ::pwrite(fd, &data[count], length - count, offset + count);
if (err <= 0)
{
err = ::pwrite(fd, &bData[count], length - count, offset + count);
if (err < 0 || (err == 0 && errno != EINTR))
{
if (count > 0)
return count;
else
return err;
}
count += err;
} while (count < length);
return count;
}
ssize_t Replicator::_write(int fd, const void *data, size_t length)
{
ssize_t err;
size_t count = 0;
uint8_t *bData = (uint8_t *) data;
do
{
err = ::write(fd, &bData[count], length - count);
if (err < 0 || (err == 0 && errno != EINTR))
{
if (count > 0)
return count;
else
return err;
}
count += err;
} while (count < length);
return count;
}
/* XXXPAT: I think we'll have to rewrite this function some; we'll have to at least clearly define
what happens in the various error scenarios.
To be more resilent in the face of hard errors, we may also want to redefine what a journal file is.
If/when we cannot fix the journal file in the face of an error, there are scenarios that the read code
will not be able to cope with. Ex, a journal entry that says it's 200 bytes long, but there are only
really 100 bytes. The read code has no way to tell the difference if there is an entry that follows
the bad entry, and that will cause an unrecoverable error.
Initial thought on a sol'n. Make each journal entry its own file in a tmp dir, ordered by a sequence
number in the filename. Then, one entry cannot affect the others, and the end of the file is unambiguously
the end of the data. On successful write, move the file to where it should be. This would also prevent
the readers from ever seeing bad data, and possibly reduce the size of some critical sections.
Benefits would be data integrity, and possibly add'l parallelism. The downside is of course, a higher
number of IO ops for the same operation.
*/
int Replicator::addJournalEntry(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length)
{
int fd, err;
uint64_t offlen[] = {(uint64_t) offset,length};
size_t count = 0;
int version = 1;
int l_errno;
char errbuf[80];
bool bHeaderChanged = false;
string headerRollback = "";
string journalFilename = msJournalPath + "/" + filename.string() + ".journal";
boost::filesystem::path firstDir = *((filename).begin());
uint64_t thisEntryMaxOffset = (offset + length - 1);
uint64_t currentMaxOffset = 0;
bool exists = boost::filesystem::exists(journalFilename);
OPEN(journalFilename.c_str(), (exists ? O_RDWR : O_WRONLY | O_CREAT))
if (!exists)
{
bHeaderChanged = true;
// create new journal file with header
string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % thisEntryMaxOffset).str();
err = _write(fd, header.c_str(), header.length() + 1);
l_errno = errno;
repHeaderDataWritten += (header.length() + 1);
if ((uint)err != (header.length() + 1))
{
// return the error because the header for this entry on a new journal file failed
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Writing journal header failed (%s).",
strerror_r(l_errno, errbuf, 80));
errno = l_errno;
return err;
}
Cache::get()->newJournalEntry(firstDir, header.length() + 1);
++replicatorJournalsCreated;
}
else
{
// read the existing header and check if max_offset needs to be updated
size_t tmp;
boost::shared_array<char> headertxt;
try
{
headertxt = seekToEndOfHeader1(fd, &tmp);
}
catch (std::runtime_error& e)
{
mpLogger->log(LOG_CRIT,"%s",e.what());
errno = EIO;
return -1;
}
catch (...)
{
mpLogger->log(LOG_CRIT,"Unknown exception caught during seekToEndOfHeader1.");
errno = EIO;
return -1;
}
stringstream ss;
ss << headertxt.get();
headerRollback = headertxt.get();
boost::property_tree::ptree header;
try
{
boost::property_tree::json_parser::read_json(ss, header);
}
catch (const boost::property_tree::json_parser::json_parser_error& e)
{
mpLogger->log(LOG_CRIT,"%s",e.what());
errno = EIO;
return -1;
}
catch (...)
{
mpLogger->log(LOG_CRIT,"Unknown exception caught during read_json.");
errno = EIO;
return -1;
}
assert(header.get<int>("version") == 1);
uint64_t currentMaxOffset = header.get<uint64_t>("max_offset");
if (thisEntryMaxOffset > currentMaxOffset)
{
bHeaderChanged = true;
string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % thisEntryMaxOffset).str();
err = _pwrite(fd, header.c_str(), header.length() + 1,0);
l_errno = errno;
repHeaderDataWritten += (header.length() + 1);
if ((uint)err != (header.length() + 1))
{
// only the header was possibly changed rollback attempt
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Updating journal header failed. "
"Attempting to rollback and continue.");
int rollbackErr = _pwrite(fd, headerRollback.c_str(), headerRollback.length() + 1,0);
if ((uint)rollbackErr == (headerRollback.length() + 1))
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header success.");
else
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed!");
errno = l_errno;
if (err < 0)
return err;
else
return 0;
}
}
}
off_t entryHeaderOffset = ::lseek(fd, 0, SEEK_END);
err = _write(fd, offlen, JOURNAL_ENTRY_HEADER_SIZE);
l_errno = errno;
repHeaderDataWritten += JOURNAL_ENTRY_HEADER_SIZE;
if (err != JOURNAL_ENTRY_HEADER_SIZE)
{
// this entry failed so if the header was updated roll it back
if (bHeaderChanged)
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: write journal entry header failed. Attempting to rollback and continue.");
//attempt to rollback top level header
int rollbackErr = _pwrite(fd, headerRollback.c_str(), headerRollback.length() + 1,0);
if ((uint)rollbackErr != (headerRollback.length() + 1))
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed! (%s)",
strerror_r(errno, errbuf, 80));
errno = l_errno;
if (err < 0)
return err;
else
return 0;
}
}
int rollbackErr = ::ftruncate(fd,entryHeaderOffset);
if (rollbackErr != 0)
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Truncate to previous EOF failed! (%s)",
strerror_r(errno, errbuf, 80));
errno = l_errno;
if (err < 0)
return err;
else
return 0;
}
l_errno = errno;
if (count > 0) // return what was successfully written
return count;
else
return err;
}
while (count < length) {
err = _write(fd, &data[count], length - count);
if (err < 0 )
{
l_errno = errno;
/* XXXBEN: Attempt to update entry header with the partial write and write it.
IF the write fails to update entry header report an error to IOC and logging.
*/
if (count > 0) // return what was successfully written
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Got '%s' writing a journal entry. "
"Attempting to update and continue.", strerror_r(l_errno, errbuf, 80));
// Update the file header max_offset if necessary and possible
thisEntryMaxOffset = (offset + count - 1);
if (thisEntryMaxOffset > currentMaxOffset)
{
string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % thisEntryMaxOffset).str();
int rollbackErr = _pwrite(fd, header.c_str(), header.length() + 1,0);
if ((uint)rollbackErr != (header.length() + 1))
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Update of journal header failed! (%s)",
strerror_r(errno, errbuf, 80));
errno = l_errno;
return err;
}
}
// Update the journal entry header
offlen[1] = count;
int rollbackErr = _pwrite(fd, offlen, JOURNAL_ENTRY_HEADER_SIZE,entryHeaderOffset);
if ((uint)rollbackErr != JOURNAL_ENTRY_HEADER_SIZE)
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Update of journal entry header failed! (%s)",
strerror_r(errno, errbuf, 80));
errno = l_errno;
return err;
}
// return back what we did write
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Partial write success.");
repUserDataWritten += count;
return count;
}
else
{
//If the header was changed rollback and reset EOF
//Like this never happened
//Currently since this returns the err from the first write. IOC returns -1 and writeTask returns an error
//So system is likely broken in some way
if (bHeaderChanged)
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: write journal entry failed (%s). "
"Attempting to rollback and continue.", strerror_r(l_errno, errbuf, 80));
//attempt to rollback top level header
string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % 0).str();
int rollbackErr = _pwrite(fd, header.c_str(), header.length() + 1,0);
if ((uint)rollbackErr != (header.length() + 1))
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed (%s)!",
strerror_r(errno, errbuf, 80));
errno = l_errno;
return err;
}
}
int rollbackErr = ::ftruncate(fd,entryHeaderOffset);
if (rollbackErr != 0)
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Remove entry header failed (%s)!",
strerror_r(errno, errbuf, 80));
errno = l_errno;
return err;
}
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Write failed. Journal file restored.");
errno = l_errno;
return err;
}
}
count += err;
}
repUserDataWritten += count;
return count;
count += err;
}
repUserDataWritten += count;
++replicatorObjectsCreated;
return count;
}
int Replicator::remove(const boost::filesystem::path &filename, Flags flags)
int Replicator::newNullObject(const boost::filesystem::path& filename, size_t length)
{
int ret = 0;
if (flags & NO_LOCAL)
return 0; // not implemented yet
int fd, err;
string objectFilename = msCachePath + "/" + filename.string();
OPEN(objectFilename.c_str(), O_WRONLY | O_CREAT);
err = ftruncate(fd, length);
return err;
}
ssize_t Replicator::_pwrite(int fd, const void* data, size_t length, off_t offset)
{
ssize_t err;
size_t count = 0;
uint8_t* bData = (uint8_t*)data;
do
{
err = ::pwrite(fd, &bData[count], length - count, offset + count);
if (err < 0 || (err == 0 && errno != EINTR))
{
if (count > 0)
return count;
else
return err;
}
count += err;
} while (count < length);
return count;
}
ssize_t Replicator::_write(int fd, const void* data, size_t length)
{
ssize_t err;
size_t count = 0;
uint8_t* bData = (uint8_t*)data;
do
{
err = ::write(fd, &bData[count], length - count);
if (err < 0 || (err == 0 && errno != EINTR))
{
if (count > 0)
return count;
else
return err;
}
count += err;
} while (count < length);
return count;
}
/* XXXPAT: I think we'll have to rewrite this function some; we'll have to at least clearly define
what happens in the various error scenarios.
To be more resilent in the face of hard errors, we may also want to redefine what a journal file is.
If/when we cannot fix the journal file in the face of an error, there are scenarios that the read code
will not be able to cope with. Ex, a journal entry that says it's 200 bytes long, but there are only
really 100 bytes. The read code has no way to tell the difference if there is an entry that follows
the bad entry, and that will cause an unrecoverable error.
Initial thought on a sol'n. Make each journal entry its own file in a tmp dir, ordered by a sequence
number in the filename. Then, one entry cannot affect the others, and the end of the file is unambiguously
the end of the data. On successful write, move the file to where it should be. This would also prevent
the readers from ever seeing bad data, and possibly reduce the size of some critical sections.
Benefits would be data integrity, and possibly add'l parallelism. The downside is of course, a higher
number of IO ops for the same operation.
*/
int Replicator::addJournalEntry(const boost::filesystem::path& filename, const uint8_t* data, off_t offset,
size_t length)
{
int fd, err;
uint64_t offlen[] = {(uint64_t)offset, length};
size_t count = 0;
int version = 1;
int l_errno;
char errbuf[80];
bool bHeaderChanged = false;
string headerRollback = "";
string journalFilename = msJournalPath + "/" + filename.string() + ".journal";
boost::filesystem::path firstDir = *((filename).begin());
uint64_t thisEntryMaxOffset = (offset + length - 1);
uint64_t currentMaxOffset = 0;
bool exists = boost::filesystem::exists(journalFilename);
OPEN(journalFilename.c_str(), (exists ? O_RDWR : O_WRONLY | O_CREAT))
if (!exists)
{
bHeaderChanged = true;
// create new journal file with header
string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version %
thisEntryMaxOffset)
.str();
err = _write(fd, header.c_str(), header.length() + 1);
l_errno = errno;
repHeaderDataWritten += (header.length() + 1);
if ((uint)err != (header.length() + 1))
{
// return the error because the header for this entry on a new journal file failed
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Writing journal header failed (%s).",
strerror_r(l_errno, errbuf, 80));
errno = l_errno;
return err;
}
Cache::get()->newJournalEntry(firstDir, header.length() + 1);
++replicatorJournalsCreated;
}
else
{
// read the existing header and check if max_offset needs to be updated
size_t tmp;
boost::shared_array<char> headertxt;
try
{
//#ifndef NDEBUG
// assert(boost::filesystem::remove_all(filename) > 0);
//#else
boost::filesystem::remove_all(filename);
//#endif
headertxt = seekToEndOfHeader1(fd, &tmp);
}
catch(boost::filesystem::filesystem_error &e)
catch (std::runtime_error& e)
{
#ifndef NDEBUG
cout << "Replicator::remove(): caught an execption: " << e.what() << endl;
assert(0);
#endif
errno = e.code().value();
ret = -1;
mpLogger->log(LOG_CRIT, "%s", e.what());
errno = EIO;
return -1;
}
return ret;
catch (...)
{
mpLogger->log(LOG_CRIT, "Unknown exception caught during seekToEndOfHeader1.");
errno = EIO;
return -1;
}
stringstream ss;
ss << headertxt.get();
headerRollback = headertxt.get();
boost::property_tree::ptree header;
try
{
boost::property_tree::json_parser::read_json(ss, header);
}
catch (const boost::property_tree::json_parser::json_parser_error& e)
{
mpLogger->log(LOG_CRIT, "%s", e.what());
errno = EIO;
return -1;
}
catch (...)
{
mpLogger->log(LOG_CRIT, "Unknown exception caught during read_json.");
errno = EIO;
return -1;
}
assert(header.get<int>("version") == 1);
uint64_t currentMaxOffset = header.get<uint64_t>("max_offset");
if (thisEntryMaxOffset > currentMaxOffset)
{
bHeaderChanged = true;
string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version %
thisEntryMaxOffset)
.str();
err = _pwrite(fd, header.c_str(), header.length() + 1, 0);
l_errno = errno;
repHeaderDataWritten += (header.length() + 1);
if ((uint)err != (header.length() + 1))
{
// only the header was possibly changed rollback attempt
mpLogger->log(LOG_CRIT,
"Replicator::addJournalEntry: Updating journal header failed. "
"Attempting to rollback and continue.");
int rollbackErr = _pwrite(fd, headerRollback.c_str(), headerRollback.length() + 1, 0);
if ((uint)rollbackErr == (headerRollback.length() + 1))
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header success.");
else
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed!");
errno = l_errno;
if (err < 0)
return err;
else
return 0;
}
}
}
off_t entryHeaderOffset = ::lseek(fd, 0, SEEK_END);
err = _write(fd, offlen, JOURNAL_ENTRY_HEADER_SIZE);
l_errno = errno;
repHeaderDataWritten += JOURNAL_ENTRY_HEADER_SIZE;
if (err != JOURNAL_ENTRY_HEADER_SIZE)
{
// this entry failed so if the header was updated roll it back
if (bHeaderChanged)
{
mpLogger->log(LOG_CRIT,
"Replicator::addJournalEntry: write journal entry header failed. Attempting to rollback "
"and continue.");
// attempt to rollback top level header
int rollbackErr = _pwrite(fd, headerRollback.c_str(), headerRollback.length() + 1, 0);
if ((uint)rollbackErr != (headerRollback.length() + 1))
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed! (%s)",
strerror_r(errno, errbuf, 80));
errno = l_errno;
if (err < 0)
return err;
else
return 0;
}
}
int rollbackErr = ::ftruncate(fd, entryHeaderOffset);
if (rollbackErr != 0)
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Truncate to previous EOF failed! (%s)",
strerror_r(errno, errbuf, 80));
errno = l_errno;
if (err < 0)
return err;
else
return 0;
}
l_errno = errno;
return err;
}
while (count < length)
{
err = _write(fd, &data[count], length - count);
if (err < 0)
{
l_errno = errno;
/* XXXBEN: Attempt to update entry header with the partial write and write it.
IF the write fails to update entry header report an error to IOC and logging.
*/
if (count > 0) // return what was successfully written
{
mpLogger->log(LOG_CRIT,
"Replicator::addJournalEntry: Got '%s' writing a journal entry. "
"Attempting to update and continue.",
strerror_r(l_errno, errbuf, 80));
// Update the file header max_offset if necessary and possible
thisEntryMaxOffset = (offset + count - 1);
if (thisEntryMaxOffset > currentMaxOffset)
{
string header = (boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version %
thisEntryMaxOffset)
.str();
int rollbackErr = _pwrite(fd, header.c_str(), header.length() + 1, 0);
if ((uint)rollbackErr != (header.length() + 1))
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Update of journal header failed! (%s)",
strerror_r(errno, errbuf, 80));
errno = l_errno;
return err;
}
}
// Update the journal entry header
offlen[1] = count;
int rollbackErr = _pwrite(fd, offlen, JOURNAL_ENTRY_HEADER_SIZE, entryHeaderOffset);
if ((uint)rollbackErr != JOURNAL_ENTRY_HEADER_SIZE)
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Update of journal entry header failed! (%s)",
strerror_r(errno, errbuf, 80));
errno = l_errno;
return err;
}
// return back what we did write
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Partial write success.");
repUserDataWritten += count;
return count;
}
else
{
// If the header was changed rollback and reset EOF
// Like this never happened
// Currently since this returns the err from the first write. IOC returns -1 and writeTask returns an
// error So system is likely broken in some way
if (bHeaderChanged)
{
mpLogger->log(LOG_CRIT,
"Replicator::addJournalEntry: write journal entry failed (%s). "
"Attempting to rollback and continue.",
strerror_r(l_errno, errbuf, 80));
// attempt to rollback top level header
string header =
(boost::format("{ \"version\" : \"%03i\", \"max_offset\" : \"%011u\" }") % version % 0).str();
int rollbackErr = _pwrite(fd, header.c_str(), header.length() + 1, 0);
if ((uint)rollbackErr != (header.length() + 1))
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Rollback of journal header failed (%s)!",
strerror_r(errno, errbuf, 80));
errno = l_errno;
return err;
}
}
int rollbackErr = ::ftruncate(fd, entryHeaderOffset);
if (rollbackErr != 0)
{
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Remove entry header failed (%s)!",
strerror_r(errno, errbuf, 80));
errno = l_errno;
return err;
}
mpLogger->log(LOG_CRIT, "Replicator::addJournalEntry: Write failed. Journal file restored.");
errno = l_errno;
return err;
}
}
count += err;
}
repUserDataWritten += count;
return count;
}
int Replicator::updateMetadata(MetadataFile &meta)
int Replicator::remove(const boost::filesystem::path& filename, Flags flags)
{
return meta.writeMetadata();
int ret = 0;
if (flags & NO_LOCAL)
return 0; // not implemented yet
try
{
//#ifndef NDEBUG
// assert(boost::filesystem::remove_all(filename) > 0);
//#else
boost::filesystem::remove_all(filename);
//#endif
}
catch (boost::filesystem::filesystem_error& e)
{
#ifndef NDEBUG
cout << "Replicator::remove(): caught an execption: " << e.what() << endl;
assert(0);
#endif
errno = e.code().value();
ret = -1;
}
return ret;
}
int Replicator::updateMetadata(MetadataFile& meta)
{
return meta.writeMetadata();
}
} // namespace storagemanager

66
storage-manager/src/Replicator.h
View File

@ -27,51 +27,49 @@
namespace storagemanager
{
// 64-bit offset
// 64-bit length
// <length-bytes of data to write at specified offset>
class Replicator
{
public:
static Replicator *get();
virtual ~Replicator();
public:
static Replicator* get();
virtual ~Replicator();
enum Flags
{
NONE = 0,
LOCAL_ONLY = 0x1,
NO_LOCAL = 0x2
};
int addJournalEntry(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length);
int newObject(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length);
int newNullObject(const boost::filesystem::path &filename,size_t length );
enum Flags
{
NONE = 0,
LOCAL_ONLY = 0x1,
NO_LOCAL = 0x2
};
int remove(const boost::filesystem::path &file, Flags flags = NONE);
int updateMetadata(MetadataFile &meta);
int addJournalEntry(const boost::filesystem::path& filename, const uint8_t* data, off_t offset,
size_t length);
int newObject(const boost::filesystem::path& filename, const uint8_t* data, off_t offset, size_t length);
int newNullObject(const boost::filesystem::path& filename, size_t length);
void printKPIs() const;
int remove(const boost::filesystem::path& file, Flags flags = NONE);
private:
Replicator();
// a couple helpers
ssize_t _write(int fd, const void *data, size_t len);
ssize_t _pwrite(int fd, const void *data, size_t len, off_t offset);
Config *mpConfig;
SMLogging *mpLogger;
std::string msJournalPath;
std::string msCachePath;
//ThreadPool threadPool;
int updateMetadata(MetadataFile& meta);
size_t repUserDataWritten, repHeaderDataWritten;
size_t replicatorObjectsCreated, replicatorJournalsCreated;
void printKPIs() const;
private:
Replicator();
// a couple helpers
ssize_t _write(int fd, const void* data, size_t len);
ssize_t _pwrite(int fd, const void* data, size_t len, off_t offset);
Config* mpConfig;
SMLogging* mpLogger;
std::string msJournalPath;
std::string msCachePath;
// ThreadPool threadPool;
size_t repUserDataWritten, repHeaderDataWritten;
size_t replicatorObjectsCreated, replicatorJournalsCreated;
};
}
} // namespace storagemanager

1241
storage-manager/src/S3Storage.cpp
View File

File diff suppressed because it is too large Load Diff

114
storage-manager/src/S3Storage.h
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <string>
@ -27,70 +26,65 @@
namespace storagemanager
{
class S3Storage : public CloudStorage
{
public:
S3Storage(bool skipRetry = false);
public:
S3Storage(bool skipRetry = false);
virtual ~S3Storage();
virtual ~S3Storage();
int getObject(const std::string &sourceKey, const std::string &destFile, size_t *size = NULL);
int getObject(const std::string &sourceKey, boost::shared_array<uint8_t> *data, size_t *size = NULL);
int putObject(const std::string &sourceFile, const std::string &destKey);
int putObject(const boost::shared_array<uint8_t> data, size_t len, const std::string &destKey);
int deleteObject(const std::string &key);
int copyObject(const std::string &sourceKey, const std::string &destKey);
int exists(const std::string &key, bool *out);
int getObject(const std::string& sourceKey, const std::string& destFile, size_t* size = NULL);
int getObject(const std::string& sourceKey, boost::shared_array<uint8_t>* data, size_t* size = NULL);
int putObject(const std::string& sourceFile, const std::string& destKey);
int putObject(const boost::shared_array<uint8_t> data, size_t len, const std::string& destKey);
int deleteObject(const std::string& key);
int copyObject(const std::string& sourceKey, const std::string& destKey);
int exists(const std::string& key, bool* out);
private:
bool getIAMRoleFromMetadataEC2();
bool getCredentialsFromMetadataEC2();
void testConnectivityAndPerms();
ms3_st *getConnection();
void returnConnection(ms3_st *);
bool skipRetryableErrors;
private:
bool getIAMRoleFromMetadataEC2();
bool getCredentialsFromMetadataEC2();
void testConnectivityAndPerms();
ms3_st* getConnection();
void returnConnection(ms3_st*);
std::string bucket; // might store this as a char *, since it's only used that way
std::string prefix;
std::string region;
std::string key;
std::string secret;
std::string token;
std::string endpoint;
std::string IAMrole;
std::string STSendpoint;
std::string STSregion;
bool isEC2Instance;
bool ec2iamEnabled;
bool useHTTP;
bool sslVerify;
int portNumber;
struct Connection
{
ms3_st *conn;
timespec idleSince;
};
struct ScopedConnection
{
ScopedConnection(S3Storage *, ms3_st *);
~ScopedConnection();
S3Storage *s3;
ms3_st *conn;
};
// for sanity checking
//std::map<ms3_st *, boost::mutex> connMutexes;
boost::mutex connMutex;
std::deque<Connection> freeConns; // using this as a stack to keep lru objects together
const time_t maxIdleSecs = 30;
bool skipRetryableErrors;
std::string bucket; // might store this as a char *, since it's only used that way
std::string prefix;
std::string region;
std::string key;
std::string secret;
std::string token;
std::string endpoint;
std::string IAMrole;
std::string STSendpoint;
std::string STSregion;
bool isEC2Instance;
bool ec2iamEnabled;
bool useHTTP;
bool sslVerify;
int portNumber;
struct Connection
{
ms3_st* conn;
timespec idleSince;
};
struct ScopedConnection
{
ScopedConnection(S3Storage*, ms3_st*);
~ScopedConnection();
S3Storage* s3;
ms3_st* conn;
};
// for sanity checking
// std::map<ms3_st *, boost::mutex> connMutexes;
boost::mutex connMutex;
std::deque<Connection> freeConns; // using this as a stack to keep lru objects together
const time_t maxIdleSecs = 30;
};
}
} // namespace storagemanager

58
storage-manager/src/SMLogging.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <stdarg.h>
#include <unistd.h>
#include "SMLogging.h"
@ -24,52 +23,51 @@ using namespace std;
namespace
{
storagemanager::SMLogging *smLog = NULL;
storagemanager::SMLogging* smLog = NULL;
boost::mutex m;
};
}; // namespace
namespace storagemanager
{
SMLogging::SMLogging()
{
//TODO: make this configurable
setlogmask (LOG_UPTO (LOG_DEBUG));
openlog ("StorageManager", LOG_PID | LOG_NDELAY | LOG_PERROR | LOG_CONS, LOG_LOCAL1);
// TODO: make this configurable
setlogmask(LOG_UPTO(LOG_DEBUG));
openlog("StorageManager", LOG_PID | LOG_NDELAY | LOG_PERROR | LOG_CONS, LOG_LOCAL1);
}
SMLogging::~SMLogging()
{
syslog(LOG_INFO, "CloseLog");
closelog();
syslog(LOG_INFO, "CloseLog");
closelog();
}
SMLogging * SMLogging::get()
SMLogging* SMLogging::get()
{
if (smLog)
return smLog;
boost::mutex::scoped_lock s(m);
if (smLog)
return smLog;
smLog = new SMLogging();
if (smLog)
return smLog;
boost::mutex::scoped_lock s(m);
if (smLog)
return smLog;
smLog = new SMLogging();
return smLog;
}
void SMLogging::log(int priority,const char *format, ...)
void SMLogging::log(int priority, const char* format, ...)
{
va_list args;
va_start(args, format);
#ifdef DEBUG
va_list args2;
va_copy(args2, args);
vfprintf(stderr, format, args2);
fprintf(stderr, "\n");
va_end(args2);
#endif
vsyslog(priority, format, args);
va_list args;
va_start(args, format);
va_end(args);
#ifdef DEBUG
va_list args2;
va_copy(args2, args);
vfprintf(stderr, format, args2);
fprintf(stderr, "\n");
va_end(args2);
#endif
vsyslog(priority, format, args);
va_end(args);
}
}
} // namespace storagemanager

22
storage-manager/src/SMLogging.h
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <syslog.h>
@ -23,22 +22,17 @@
namespace storagemanager
{
class SMLogging
{
public:
public:
static SMLogging* get();
~SMLogging();
static SMLogging *get();
~SMLogging();
void log(int priority, const char *format, ...);
private:
SMLogging();
//SMConfig& config;
void log(int priority, const char* format, ...);
private:
SMLogging();
// SMConfig& config;
};
}
} // namespace storagemanager

828
storage-manager/src/SessionManager.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
@ -41,449 +40,450 @@ using namespace std;
namespace
{
storagemanager::SessionManager *sm = NULL;
boost::mutex m;
}
storagemanager::SessionManager* sm = NULL;
boost::mutex m;
} // namespace
namespace storagemanager
{
SessionManager::SessionManager()
{
crp = ClientRequestProcessor::get();
socketCtrl[0] = -1;
socketCtrl[1] = -1;
crp = ClientRequestProcessor::get();
socketCtrl[0] = -1;
socketCtrl[1] = -1;
}
SessionManager::~SessionManager()
{
}
SessionManager * SessionManager::get()
SessionManager* SessionManager::get()
{
if (sm)
return sm;
boost::mutex::scoped_lock s(m);
if (sm)
return sm;
sm = new SessionManager();
if (sm)
return sm;
boost::mutex::scoped_lock s(m);
if (sm)
return sm;
sm = new SessionManager();
return sm;
}
int SessionManager::start()
{
SMLogging* logger = SMLogging::get();
int rc,listenSockfd,incomingSockfd,on = 1;
struct sockaddr_un addr;
int nfds;
int pollTimeout = -1;
int socketIncr;
int current_size = 0;
SMLogging* logger = SMLogging::get();
int rc, listenSockfd, incomingSockfd, on = 1;
struct sockaddr_un addr;
int nfds;
int pollTimeout = -1;
int socketIncr;
int current_size = 0;
bool shutdown = false;
bool running = true;
bool shutdown = false;
bool running = true;
if (pipe(socketCtrl)==-1)
{
logger->log(LOG_CRIT,"Pipe Failed: %s", strerror(errno));
return 1;
}
listenSockfd = ::socket(AF_UNIX, SOCK_STREAM, 0);
if (listenSockfd < 0)
{
logger->log(LOG_CRIT,"socket() failed: %s", strerror(errno));
return -1;
}
rc = ::setsockopt(listenSockfd, SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on));
if (rc < 0)
{
logger->log(LOG_CRIT,"setsockopt() failed: %s", strerror(errno));
close(listenSockfd);
return -1;
}
rc = ::ioctl(listenSockfd, FIONBIO, (char *)&on);
if (rc < 0)
{
logger->log(LOG_CRIT,"ioctl() failed: %s", strerror(errno));
close(listenSockfd);
return -1;
}
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &storagemanager::socket_name[1]); // first char is null...
rc = ::bind(listenSockfd,(struct sockaddr *)&addr, sizeof(addr));
if (rc < 0)
{
logger->log(LOG_CRIT,"bind() failed: %s", strerror(errno));
close(listenSockfd);
return -1;
}
rc = ::listen(listenSockfd, 32);
if (rc < 0)
{
logger->log(LOG_CRIT,"listen() failed: %s", strerror(errno));
close(listenSockfd);
return -1;
}
memset(fds, 0 , sizeof(fds));
fds[0].fd = listenSockfd;
fds[0].events = POLLIN;
fds[1].fd = socketCtrl[0];
fds[1].events = POLLIN;
nfds = 2;
logger->log(LOG_INFO,"SessionManager waiting for sockets.");
while (running)
{
try
{
//if (current_size != nfds)
//logger->log(LOG_DEBUG,"polling %i fds %i", nfds,fds);
//cout << "polling " << nfds << " fds" << endl;
rc = ::poll(fds, nfds, pollTimeout);
if (rc < 0 && errno != EINTR)
{
logger->log(LOG_CRIT,"poll() failed: %s", strerror(errno));
break;
}
else if (rc < 0)
continue;
current_size = nfds;
for (socketIncr = 0; socketIncr < current_size; socketIncr++)
{
if(fds[socketIncr].revents == 0)
continue;
//if (socketIncr >= 2)
//logger->log(LOG_DEBUG,"got event on fd %i index is %i", fds[socketIncr].fd,socketIncr);
if(fds[socketIncr].revents != POLLIN)
{
//logger->log(LOG_DEBUG,"Error! revents = %d", fds[socketIncr].revents,);
if (fds[socketIncr].fd == -1)
logger->log(LOG_DEBUG,"!= POLLIN, closing fd -1");
close(fds[socketIncr].fd);
fds[socketIncr].fd = -1;
continue;
}
if (fds[socketIncr].fd == listenSockfd)
{
//logger->log(LOG_DEBUG,"Listening socket is readable");
incomingSockfd = 0;
while (incomingSockfd != -1 && !shutdown)
{
if (nfds >= MAX_SM_SOCKETS)
break; // try to free up some room
incomingSockfd = ::accept(listenSockfd, NULL, NULL);
if (incomingSockfd < 0)
{
if (errno != EWOULDBLOCK)
{
logger->log(LOG_CRIT,"accept() failed: %s", strerror(errno));
shutdown = true;
}
break;
}
//logger->log(LOG_DEBUG,"New incoming connection - %d",incomingSockfd);
fds[nfds].fd = incomingSockfd;
fds[nfds].events = POLLIN;
nfds++;
}
}
else if (fds[socketIncr].fd == socketCtrl[0])
{
//logger->log(LOG_DEBUG,"SocketControl is readable");
uint8_t ctrlCode;
int len,socket;
len = ::read(socketCtrl[0], &ctrlCode, 1);
if (len <= 0)
{
continue;
}
switch(ctrlCode)
{
case ADDFD:
len = ::read(socketCtrl[0], &socket, sizeof(socket));
if (len <= 0)
{
continue;
}
for (int i = 0; i < nfds; i++)
{
if(fds[i].fd == socket)
{
if (shutdown)
{
logger->log(LOG_DEBUG,"Shutdown in progress, closed socket %i at index %i", fds[i].fd,i);
close(socket);
break;
}
fds[i].events = (POLLIN | POLLPRI);
break;
}
}
break;
case REMOVEFD:
len = ::read(socketCtrl[0], &socket, sizeof(socket));
if (len <= 0)
{
continue;
}
for (int i = 0; i < nfds; i++)
{
if (fds[i].fd == socket)
{
if (socket == -1)
logger->log(LOG_DEBUG,"REMOVEFD told to remove fd -1");
close(socket);
fds[i].fd = -1;
break;
}
}
break;
case SHUTDOWN:
logger->log(LOG_DEBUG,"Shutdown StorageManager...");
close(fds[0].fd);
shutdown = true;
break;
default:
break;
}
}
else
{
//logger->log(LOG_DEBUG,"socketIncr %d -- Descriptor %d is readable",socketIncr,fds[socketIncr].fd);
bool closeConn = false;
char recv_buffer[8192];
uint recvMsgLength = 0;
uint recvMsgStart = 0;
uint remainingBytes = 0;
uint endOfData, i;
int peakLength,len;
//logger->log(LOG_DEBUG,"reading from fd %i index is %i", fds[socketIncr].fd,socketIncr);
if (sockState.find(fds[socketIncr].fd) != sockState.end())
{
SockState &state = sockState[fds[socketIncr].fd];
remainingBytes = state.remainingBytes;
memcpy(recv_buffer, state.remainingData, remainingBytes);
sockState.erase(fds[socketIncr].fd);
}
while(true)
{
peakLength = ::recv(fds[socketIncr].fd, &recv_buffer[remainingBytes], sizeof(recv_buffer)-remainingBytes, MSG_PEEK | MSG_DONTWAIT);
if (peakLength < 0)
{
if (errno != EWOULDBLOCK)
{
closeConn = true;
break;
}
// there's no data available at this point; go back to the poll loop.
assert(remainingBytes < SM_HEADER_LEN);
SockState state;
state.remainingBytes = remainingBytes;
memcpy(state.remainingData, recv_buffer, remainingBytes);
sockState[fds[socketIncr].fd] = state;
break;
}
//logger->log(LOG_DEBUG,"recv got %i bytes", peakLength);
endOfData = remainingBytes + peakLength;
if (endOfData < SM_HEADER_LEN)
{
//read this snippet and keep going
len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], peakLength);
remainingBytes = endOfData;
if (len != peakLength)
logger->log(LOG_ERR,"Read returned length != peakLength. ( %i != %i )", len, peakLength);
continue;
}
//Look for SM_MSG_START
for (i = 0; i <= endOfData - SM_HEADER_LEN; i++)
{
if (*((uint *) &recv_buffer[i]) == SM_MSG_START)
{
//logger->log(LOG_DEBUG,"Received SM_MSG_START");
//found it set msgLength and recvMsgStart offset of SM_MSG_START
recvMsgLength = *((uint *) &recv_buffer[i+4]);
//logger->log(LOG_DEBUG,"got length = %i", recvMsgLength);
recvMsgStart = i + SM_HEADER_LEN;
//logger->log(LOG_DEBUG,"recvMsgLength %d recvMsgStart %d endofData %d", recvMsgLength,recvMsgStart,endOfData);
// if >= endOfData then the start of the message data is the beginning of next message
if (recvMsgStart >= endOfData)
recvMsgStart = 0;
break;
}
}
// didn't find SM_MSG_START in this message consume the data and loop back through on next message
if (recvMsgLength == 0)
{
//logger->log(LOG_DEBUG,"No SM_MSG_START");
len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], peakLength);
if (len != peakLength)
logger->log(LOG_ERR,"Read returned length != peakLength. ( %i != %i )", len, peakLength);
// we know the msg header isn't in position [0, endOfData - i), so throw that out
// and copy [i, endOfData) to the front of the buffer to be
// checked by the next iteration.
memcpy(recv_buffer, &recv_buffer[i], endOfData - i);
remainingBytes = endOfData - i;
}
else
{
//found msg start
//remove the junk in front of the message
if (recvMsgStart > 0)
{
//logger->log(LOG_DEBUG,"SM_MSG_START data is here");
// how many to consume here...
// recvMsgStart is the position in the buffer
// peakLength is the amount peeked this time
// remainingBytes is the amount carried over from previous reads
len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], recvMsgStart - remainingBytes);
}
else
{
//logger->log(LOG_DEBUG,"SM_MSG_START data is next message");
len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], peakLength);
if (len != peakLength)
logger->log(LOG_ERR,"Read returned length != peakLength. ( %i != %i )", len, peakLength);
}
//Disable polling on this socket
fds[socketIncr].events = 0;
//Pass the socket to CRP and the message length. next read will be start of message
crp->processRequest(fds[socketIncr].fd,recvMsgLength);
/*
//Doing this to work with cloudio_component_test
len = ::read(fds[socketIncr].fd, out, recvMsgLength);
logger->log(LOG_DEBUG,"Read %d bytes.",len);
//Debug test lets send a reponse back
uint32_t response[4] = { storagemanager::SM_MSG_START, 8, (uint32_t ) -1, EINVAL };
len = ::send(fds[socketIncr].fd, response, 16, 0);
*/
break;
}
}
if (closeConn)
{
if (fds[socketIncr].fd == -1)
logger->log(LOG_DEBUG,"closeConn closing fd -1");
close(fds[socketIncr].fd);
fds[socketIncr].fd = -1;
}
}
}
}
catch (std::exception& ex)
{
throw std::exception();
}
catch (...)
{
throw std::exception();
}
int i, j;
if (shutdown)
{
for (i = 2; i < nfds; ++i)
{
if (fds[i].events == (POLLIN | POLLPRI))
{
close(fds[i].fd);
fds[i].fd = -1;
}
}
}
/* get rid of fds == -1 */
for (i = 2; i < nfds; ++i)
{
if (fds[i].fd == -1)
break;
}
for (j = i + 1; j < nfds; ++j)
{
if (fds[j].fd != -1)
{
fds[i].fd = fds[j].fd;
fds[i].events = fds[j].events;
++i;
}
}
nfds = i;
if(shutdown)
{
if (nfds <= 2)
running = false;
}
}
//Shutdown Done
crp->shutdown();
if (pipe(socketCtrl) == -1)
{
logger->log(LOG_CRIT, "Pipe Failed: %s", strerror(errno));
return 1;
}
listenSockfd = ::socket(AF_UNIX, SOCK_STREAM, 0);
if (listenSockfd < 0)
{
logger->log(LOG_CRIT, "socket() failed: %s", strerror(errno));
return -1;
}
rc = ::setsockopt(listenSockfd, SOL_SOCKET, SO_REUSEADDR, (char*)&on, sizeof(on));
if (rc < 0)
{
logger->log(LOG_CRIT, "setsockopt() failed: %s", strerror(errno));
close(listenSockfd);
return -1;
}
rc = ::ioctl(listenSockfd, FIONBIO, (char*)&on);
if (rc < 0)
{
logger->log(LOG_CRIT, "ioctl() failed: %s", strerror(errno));
close(listenSockfd);
return -1;
}
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &storagemanager::socket_name[1]); // first char is null...
rc = ::bind(listenSockfd, (struct sockaddr*)&addr, sizeof(addr));
if (rc < 0)
{
logger->log(LOG_CRIT, "bind() failed: %s", strerror(errno));
close(listenSockfd);
return -1;
}
rc = ::listen(listenSockfd, 32);
if (rc < 0)
{
logger->log(LOG_CRIT, "listen() failed: %s", strerror(errno));
close(listenSockfd);
return -1;
}
memset(fds, 0, sizeof(fds));
fds[0].fd = listenSockfd;
fds[0].events = POLLIN;
fds[1].fd = socketCtrl[0];
fds[1].events = POLLIN;
nfds = 2;
logger->log(LOG_INFO, "SessionManager waiting for sockets.");
while (running)
{
try
{
// if (current_size != nfds)
// logger->log(LOG_DEBUG,"polling %i fds %i", nfds,fds);
// cout << "polling " << nfds << " fds" << endl;
rc = ::poll(fds, nfds, pollTimeout);
if (rc < 0 && errno != EINTR)
{
logger->log(LOG_CRIT, "poll() failed: %s", strerror(errno));
break;
}
else if (rc < 0)
continue;
current_size = nfds;
for (socketIncr = 0; socketIncr < current_size; socketIncr++)
{
if (fds[socketIncr].revents == 0)
continue;
// if (socketIncr >= 2)
// logger->log(LOG_DEBUG,"got event on fd %i index is %i", fds[socketIncr].fd,socketIncr);
if (fds[socketIncr].revents != POLLIN)
{
// logger->log(LOG_DEBUG,"Error! revents = %d", fds[socketIncr].revents,);
if (fds[socketIncr].fd == -1)
logger->log(LOG_DEBUG, "!= POLLIN, closing fd -1");
close(fds[socketIncr].fd);
fds[socketIncr].fd = -1;
continue;
}
if (fds[socketIncr].fd == listenSockfd)
{
// logger->log(LOG_DEBUG,"Listening socket is readable");
incomingSockfd = 0;
while (incomingSockfd != -1 && !shutdown)
{
if (nfds >= MAX_SM_SOCKETS)
break; // try to free up some room
incomingSockfd = ::accept(listenSockfd, NULL, NULL);
if (incomingSockfd < 0)
{
if (errno != EWOULDBLOCK)
{
logger->log(LOG_CRIT, "accept() failed: %s", strerror(errno));
shutdown = true;
}
break;
}
// logger->log(LOG_DEBUG,"New incoming connection - %d",incomingSockfd);
fds[nfds].fd = incomingSockfd;
fds[nfds].events = POLLIN;
nfds++;
}
}
else if (fds[socketIncr].fd == socketCtrl[0])
{
// logger->log(LOG_DEBUG,"SocketControl is readable");
uint8_t ctrlCode;
int len, socket;
len = ::read(socketCtrl[0], &ctrlCode, 1);
if (len <= 0)
{
continue;
}
switch (ctrlCode)
{
case ADDFD:
len = ::read(socketCtrl[0], &socket, sizeof(socket));
if (len <= 0)
{
continue;
}
for (int i = 0; i < nfds; i++)
{
if (fds[i].fd == socket)
{
if (shutdown)
{
logger->log(LOG_DEBUG, "Shutdown in progress, closed socket %i at index %i", fds[i].fd,
i);
close(socket);
break;
}
fds[i].events = (POLLIN | POLLPRI);
break;
}
}
break;
case REMOVEFD:
len = ::read(socketCtrl[0], &socket, sizeof(socket));
if (len <= 0)
{
continue;
}
for (int i = 0; i < nfds; i++)
{
if (fds[i].fd == socket)
{
if (socket == -1)
logger->log(LOG_DEBUG, "REMOVEFD told to remove fd -1");
close(socket);
fds[i].fd = -1;
break;
}
}
break;
case SHUTDOWN:
logger->log(LOG_DEBUG, "Shutdown StorageManager...");
close(fds[0].fd);
shutdown = true;
break;
default: break;
}
}
else
{
// logger->log(LOG_DEBUG,"socketIncr %d -- Descriptor %d is
// readable",socketIncr,fds[socketIncr].fd);
bool closeConn = false;
char recv_buffer[8192];
uint recvMsgLength = 0;
uint recvMsgStart = 0;
uint remainingBytes = 0;
uint endOfData, i;
int peakLength, len;
// logger->log(LOG_DEBUG,"reading from fd %i index is %i", fds[socketIncr].fd,socketIncr);
if (sockState.find(fds[socketIncr].fd) != sockState.end())
{
SockState& state = sockState[fds[socketIncr].fd];
remainingBytes = state.remainingBytes;
memcpy(recv_buffer, state.remainingData, remainingBytes);
sockState.erase(fds[socketIncr].fd);
}
while (true)
{
peakLength = ::recv(fds[socketIncr].fd, &recv_buffer[remainingBytes],
sizeof(recv_buffer) - remainingBytes, MSG_PEEK | MSG_DONTWAIT);
if (peakLength < 0)
{
if (errno != EWOULDBLOCK)
{
closeConn = true;
break;
}
// there's no data available at this point; go back to the poll loop.
assert(remainingBytes < SM_HEADER_LEN);
SockState state;
state.remainingBytes = remainingBytes;
memcpy(state.remainingData, recv_buffer, remainingBytes);
sockState[fds[socketIncr].fd] = state;
break;
}
// logger->log(LOG_DEBUG,"recv got %i bytes", peakLength);
endOfData = remainingBytes + peakLength;
if (endOfData < SM_HEADER_LEN)
{
// read this snippet and keep going
len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], peakLength);
remainingBytes = endOfData;
if (len != peakLength)
logger->log(LOG_ERR, "Read returned length != peakLength. ( %i != %i )", len, peakLength);
continue;
}
// Look for SM_MSG_START
for (i = 0; i <= endOfData - SM_HEADER_LEN; i++)
{
if (*((uint*)&recv_buffer[i]) == SM_MSG_START)
{
// logger->log(LOG_DEBUG,"Received SM_MSG_START");
// found it set msgLength and recvMsgStart offset of SM_MSG_START
recvMsgLength = *((uint*)&recv_buffer[i + 4]);
// logger->log(LOG_DEBUG,"got length = %i", recvMsgLength);
recvMsgStart = i + SM_HEADER_LEN;
// logger->log(LOG_DEBUG,"recvMsgLength %d recvMsgStart %d endofData %d",
// recvMsgLength,recvMsgStart,endOfData);
// if >= endOfData then the start of the message data is the beginning of next message
if (recvMsgStart >= endOfData)
recvMsgStart = 0;
break;
}
}
// didn't find SM_MSG_START in this message consume the data and loop back through on next message
if (recvMsgLength == 0)
{
// logger->log(LOG_DEBUG,"No SM_MSG_START");
len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], peakLength);
if (len != peakLength)
logger->log(LOG_ERR, "Read returned length != peakLength. ( %i != %i )", len, peakLength);
// we know the msg header isn't in position [0, endOfData - i), so throw that out
// and copy [i, endOfData) to the front of the buffer to be
// checked by the next iteration.
memcpy(recv_buffer, &recv_buffer[i], endOfData - i);
remainingBytes = endOfData - i;
}
else
{
// found msg start
// remove the junk in front of the message
if (recvMsgStart > 0)
{
// logger->log(LOG_DEBUG,"SM_MSG_START data is here");
// how many to consume here...
// recvMsgStart is the position in the buffer
// peakLength is the amount peeked this time
// remainingBytes is the amount carried over from previous reads
len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], recvMsgStart - remainingBytes);
}
else
{
// logger->log(LOG_DEBUG,"SM_MSG_START data is next message");
len = ::read(fds[socketIncr].fd, &recv_buffer[remainingBytes], peakLength);
if (len != peakLength)
logger->log(LOG_ERR, "Read returned length != peakLength. ( %i != %i )", len, peakLength);
}
// Disable polling on this socket
fds[socketIncr].events = 0;
// Pass the socket to CRP and the message length. next read will be start of message
crp->processRequest(fds[socketIncr].fd, recvMsgLength);
/*
//Doing this to work with cloudio_component_test
len = ::read(fds[socketIncr].fd, out, recvMsgLength);
logger->log(LOG_DEBUG,"Read %d bytes.",len);
//Debug test lets send a reponse back
uint32_t response[4] = { storagemanager::SM_MSG_START, 8, (uint32_t ) -1, EINVAL };
len = ::send(fds[socketIncr].fd, response, 16, 0);
*/
break;
}
}
if (closeConn)
{
if (fds[socketIncr].fd == -1)
logger->log(LOG_DEBUG, "closeConn closing fd -1");
close(fds[socketIncr].fd);
fds[socketIncr].fd = -1;
}
}
}
}
catch (std::exception& ex)
{
throw std::exception();
}
catch (...)
{
throw std::exception();
}
int i, j;
if (shutdown)
{
for (i = 2; i < nfds; ++i)
{
if (fds[i].events == (POLLIN | POLLPRI))
{
close(fds[i].fd);
fds[i].fd = -1;
}
}
}
/* get rid of fds == -1 */
for (i = 2; i < nfds; ++i)
{
if (fds[i].fd == -1)
break;
}
for (j = i + 1; j < nfds; ++j)
{
if (fds[j].fd != -1)
{
fds[i].fd = fds[j].fd;
fds[i].events = fds[j].events;
++i;
}
}
nfds = i;
if (shutdown)
{
if (nfds <= 2)
running = false;
}
}
// Shutdown Done
crp->shutdown();
return -1;
}
void SessionManager::returnSocket(int socket)
{
boost::mutex::scoped_lock s(ctrlMutex);
int err;
uint8_t ctrlCode = ADDFD;
err = ::write(socketCtrl[1], &ctrlCode, 1);
if (err <= 0)
{
return;
}
err = ::write(socketCtrl[1], &socket, sizeof(socket));
if (err <= 0)
{
return;
}
boost::mutex::scoped_lock s(ctrlMutex);
int err;
uint8_t ctrlCode = ADDFD;
err = ::write(socketCtrl[1], &ctrlCode, 1);
if (err <= 0)
{
return;
}
err = ::write(socketCtrl[1], &socket, sizeof(socket));
if (err <= 0)
{
return;
}
}
void SessionManager::socketError(int socket)
{
boost::mutex::scoped_lock s(ctrlMutex);
SMLogging* logger = SMLogging::get();
logger->log(LOG_CRIT," ****** socket error!");
int err;
uint8_t ctrlCode = REMOVEFD;
err = ::write(socketCtrl[1], &ctrlCode, 1);
if (err <= 0)
{
return;
}
err = ::write(socketCtrl[1], &socket, sizeof(socket));
if (err <= 0)
{
return;
}
}
void SessionManager::shutdownSM(int sig){
boost::mutex::scoped_lock s(ctrlMutex);
SMLogging* logger = SMLogging::get();
logger->log(LOG_DEBUG,"SessionManager Caught Signal %i",sig);
int err;
uint8_t ctrlCode = SHUTDOWN;
err = ::write(socketCtrl[1], &ctrlCode, 1);
if (err <= 0)
{
return;
}
boost::mutex::scoped_lock s(ctrlMutex);
SMLogging* logger = SMLogging::get();
logger->log(LOG_CRIT, " ****** socket error!");
int err;
uint8_t ctrlCode = REMOVEFD;
err = ::write(socketCtrl[1], &ctrlCode, 1);
if (err <= 0)
{
return;
}
err = ::write(socketCtrl[1], &socket, sizeof(socket));
if (err <= 0)
{
return;
}
}
void SessionManager::shutdownSM(int sig)
{
boost::mutex::scoped_lock s(ctrlMutex);
SMLogging* logger = SMLogging::get();
logger->log(LOG_DEBUG, "SessionManager Caught Signal %i", sig);
int err;
uint8_t ctrlCode = SHUTDOWN;
err = ::write(socketCtrl[1], &ctrlCode, 1);
if (err <= 0)
{
return;
}
}
} // namespace storagemanager

70
storage-manager/src/SessionManager.h
View File

@ -15,8 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "ClientRequestProcessor.h"
@ -27,14 +25,13 @@
#include <sys/poll.h>
#include <tr1/unordered_map>
namespace storagemanager
{
enum sessionCtrl {
ADDFD,
REMOVEFD,
SHUTDOWN
enum sessionCtrl
{
ADDFD,
REMOVEFD,
SHUTDOWN
};
#define MAX_SM_SOCKETS 200
@ -44,37 +41,36 @@ enum sessionCtrl {
*/
class SessionManager
{
public:
static SessionManager *get();
~SessionManager();
public:
static SessionManager* get();
~SessionManager();
/**
* start and manage socket connections
*/
int start();
/**
* start and manage socket connections
*/
int start();
void returnSocket(int socket);
void socketError(int socket);
void CRPTest(int socket,uint length);
void shutdownSM(int sig);
void returnSocket(int socket);
void socketError(int socket);
void CRPTest(int socket, uint length);
void shutdownSM(int sig);
private:
SessionManager();
//SMConfig& config;
ClientRequestProcessor *crp;
struct pollfd fds[MAX_SM_SOCKETS];
int socketCtrl[2];
boost::mutex ctrlMutex;
// These map a socket fd to its state between read iterations if a message header could not be found in the data
// available at the time.
struct SockState {
char remainingData[SM_HEADER_LEN];
uint remainingBytes;
};
std::tr1::unordered_map<int, SockState> sockState;
private:
SessionManager();
// SMConfig& config;
ClientRequestProcessor* crp;
struct pollfd fds[MAX_SM_SOCKETS];
int socketCtrl[2];
boost::mutex ctrlMutex;
// These map a socket fd to its state between read iterations if a message header could not be found in the
// data available at the time.
struct SockState
{
char remainingData[SM_HEADER_LEN];
uint remainingBytes;
};
std::tr1::unordered_map<int, SockState> sockState;
};
}
} // namespace storagemanager

91
storage-manager/src/StatTask.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "StatTask.h"
#include "messageFormat.h"
#include "SMLogging.h"
@ -29,7 +28,6 @@ using namespace std;
namespace storagemanager
{
StatTask::StatTask(int sock, uint len) : PosixTask(sock, len)
{
}
@ -39,53 +37,54 @@ StatTask::~StatTask()
}
#define check_error(msg, ret) \
if (success<0) \
{ \
handleError(msg, errno); \
return ret; \
}
if (success < 0) \
{ \
handleError(msg, errno); \
return ret; \
}
bool StatTask::run()
{
SMLogging* logger = SMLogging::get();
int success;
uint8_t buf[1024] = {0};
if (getLength() > 1023) {
handleError("StatTask read", ENAMETOOLONG);
return true;
}
success = read(buf, getLength());
check_error("StatTask read", false);
stat_cmd *cmd = (stat_cmd *) buf;
sm_response *resp = (sm_response *) buf;
#ifdef SM_TRACE
logger->log(LOG_DEBUG,"stat %s.",cmd->filename);
#endif
int err;
try
{
err = ioc->stat(cmd->filename, (struct stat *) resp->payload);
}
catch (exception &e)
{
logger->log(LOG_ERR, "StatTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
resp->returnCode = err;
uint payloadLen;
if (!err)
payloadLen = sizeof(struct stat);
else {
payloadLen = 4;
*((int32_t *) resp->payload) = errno;
}
return write(*resp, payloadLen);
}
SMLogging* logger = SMLogging::get();
int success;
uint8_t buf[1024] = {0};
if (getLength() > 1023)
{
handleError("StatTask read", ENAMETOOLONG);
return true;
}
success = read(buf, getLength());
check_error("StatTask read", false);
stat_cmd* cmd = (stat_cmd*)buf;
sm_response* resp = (sm_response*)buf;
#ifdef SM_TRACE
logger->log(LOG_DEBUG, "stat %s.", cmd->filename);
#endif
int err;
try
{
err = ioc->stat(cmd->filename, (struct stat*)resp->payload);
}
catch (exception& e)
{
logger->log(LOG_ERR, "StatTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
resp->returnCode = err;
uint payloadLen;
if (!err)
payloadLen = sizeof(struct stat);
else
{
payloadLen = 4;
*((int32_t*)resp->payload) = errno;
}
return write(*resp, payloadLen);
}
} // namespace storagemanager

21
storage-manager/src/StatTask.h
View File

@ -15,25 +15,22 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "PosixTask.h"
namespace storagemanager
{
class StatTask : public PosixTask
{
public:
StatTask(int sock, uint length);
virtual ~StatTask();
bool run();
private:
StatTask();
public:
StatTask(int sock, uint length);
virtual ~StatTask();
bool run();
private:
StatTask();
};
}
} // namespace storagemanager

44
storage-manager/src/SyncTask.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "SyncTask.h"
#include "Synchronizer.h"
#include "messageFormat.h"
@ -23,7 +22,6 @@
namespace storagemanager
{
SyncTask::SyncTask(int sock, uint len) : PosixTask(sock, len)
{
}
@ -34,29 +32,29 @@ SyncTask::~SyncTask()
bool SyncTask::run()
{
// force flush synchronizer
uint8_t buf;
// force flush synchronizer
uint8_t buf;
if (getLength() > 1)
{
handleError("SyncTask", E2BIG);
return true;
}
// consume the msg
int success = read(&buf, getLength());
if (success<0)
{
handleError("SyncTask", errno);
return false;
}
if (getLength() > 1)
{
handleError("SyncTask", E2BIG);
return true;
}
// consume the msg
int success = read(&buf, getLength());
if (success < 0)
{
handleError("SyncTask", errno);
return false;
}
Synchronizer::get()->syncNow();
Synchronizer::get()->syncNow();
// send generic success response
sm_response ret;
ret.returnCode = 0;
success = write(ret, 0);
return success;
// send generic success response
sm_response ret;
ret.returnCode = 0;
success = write(ret, 0);
return success;
}
}
} // namespace storagemanager

17
storage-manager/src/SyncTask.h
View File

@ -15,25 +15,22 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "PosixTask.h"
namespace storagemanager
{
class SyncTask : public PosixTask
{
public:
SyncTask(int sock, uint length);
virtual ~SyncTask();
public:
SyncTask(int sock, uint length);
virtual ~SyncTask();
bool run();
bool run();
private:
SyncTask();
private:
SyncTask();
};
}
} // namespace storagemanager

1466
storage-manager/src/Synchronizer.cpp
View File

File diff suppressed because it is too large Load Diff

187
storage-manager/src/Synchronizer.h
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <string>
@ -34,106 +33,106 @@
namespace storagemanager
{
class Cache; // break circular dependency in header files
class IOCoordinator;
class Synchronizer : public boost::noncopyable , public ConfigListener
class Synchronizer : public boost::noncopyable, public ConfigListener
{
public:
static Synchronizer *get();
virtual ~Synchronizer();
// these take keys as parameters, not full path names, ex, pass in '12345' not
// 'cache/12345'.
void newJournalEntry(const boost::filesystem::path &firstDir, const std::string &key, size_t len);
void newJournalEntries(const boost::filesystem::path &firstDir, const std::vector<std::pair<std::string, size_t> > &keys);
void newObjects(const boost::filesystem::path &firstDir, const std::vector<std::string> &keys);
void deletedObjects(const boost::filesystem::path &firstDir, const std::vector<std::string> &keys);
void flushObject(const boost::filesystem::path &firstDir, const std::string &key);
void forceFlush(); // ideally, make a version of this that takes a firstDir parameter
void syncNow(); // synchronous version of force for SyncTask
public:
static Synchronizer* get();
virtual ~Synchronizer();
void newPrefix(const boost::filesystem::path &p);
void dropPrefix(const boost::filesystem::path &p);
// for testing primarily
boost::filesystem::path getJournalPath();
boost::filesystem::path getCachePath();
void printKPIs() const;
// these take keys as parameters, not full path names, ex, pass in '12345' not
// 'cache/12345'.
void newJournalEntry(const boost::filesystem::path& firstDir, const std::string& key, size_t len);
void newJournalEntries(const boost::filesystem::path& firstDir,
const std::vector<std::pair<std::string, size_t> >& keys);
void newObjects(const boost::filesystem::path& firstDir, const std::vector<std::string>& keys);
void deletedObjects(const boost::filesystem::path& firstDir, const std::vector<std::string>& keys);
void flushObject(const boost::filesystem::path& firstDir, const std::string& key);
void forceFlush(); // ideally, make a version of this that takes a firstDir parameter
void syncNow(); // synchronous version of force for SyncTask
virtual void configListener() override;
void newPrefix(const boost::filesystem::path& p);
void dropPrefix(const boost::filesystem::path& p);
private:
Synchronizer();
void _newJournalEntry(const boost::filesystem::path &firstDir, const std::string &key, size_t len);
void process(std::list<std::string>::iterator key);
void synchronize(const std::string &sourceFile, std::list<std::string>::iterator &it);
void synchronizeDelete(const std::string &sourceFile, std::list<std::string>::iterator &it);
void synchronizeWithJournal(const std::string &sourceFile, std::list<std::string>::iterator &it);
void rename(const std::string &oldkey, const std::string &newkey);
void makeJob(const std::string &key);
// this struct kind of got sloppy. Need to clean it at some point.
struct PendingOps
{
PendingOps(int flags);
~PendingOps();
int opFlags;
int waiters;
bool finished;
boost::condition condvar;
void wait(boost::mutex *);
void notify();
};
struct Job : public ThreadPool::Job
{
virtual ~Job(){};
Job(Synchronizer *s, std::list<std::string>::iterator i);
void operator()();
Synchronizer *sync;
std::list<std::string>::iterator it;
};
uint maxUploads;
boost::scoped_ptr<ThreadPool> threadPool;
std::map<std::string, boost::shared_ptr<PendingOps> > pendingOps;
std::map<std::string, boost::shared_ptr<PendingOps> > opsInProgress;
// this is a bit of a kludge to handle objects being renamed. Jobs can be issued
// against name1, but when the job starts running, the target may be name2.
// some consolidation should be possible between this and the two maps above, tbd.
// in general the code got kludgier b/c of renaming, needs a cleanup pass.
std::list<std::string> objNames;
// this thread will start jobs for entries in pendingOps every 10 seconds
bool die;
boost::thread syncThread;
const boost::chrono::seconds syncInterval = boost::chrono::seconds(10);
void periodicSync();
std::map<boost::filesystem::path, size_t> uncommittedJournalSize;
size_t journalSizeThreshold;
bool blockNewJobs;
void syncNow(const boost::filesystem::path &prefix); // a synchronous version of forceFlush()
// for testing primarily
boost::filesystem::path getJournalPath();
boost::filesystem::path getCachePath();
void printKPIs() const;
// some KPIs
size_t numBytesRead, numBytesWritten, numBytesUploaded, numBytesDownloaded,
flushesTriggeredBySize, flushesTriggeredByTimer, journalsMerged, objectsSyncedWithNoJournal,
bytesReadBySync, bytesReadBySyncWithJournal;
ssize_t mergeDiff;
SMLogging *logger;
Cache *cache;
Replicator *replicator;
IOCoordinator *ioc;
CloudStorage *cs;
boost::filesystem::path cachePath;
boost::filesystem::path journalPath;
boost::mutex mutex;
virtual void configListener() override;
private:
Synchronizer();
void _newJournalEntry(const boost::filesystem::path& firstDir, const std::string& key, size_t len);
void process(std::list<std::string>::iterator key);
void synchronize(const std::string& sourceFile, std::list<std::string>::iterator& it);
void synchronizeDelete(const std::string& sourceFile, std::list<std::string>::iterator& it);
void synchronizeWithJournal(const std::string& sourceFile, std::list<std::string>::iterator& it);
void rename(const std::string& oldkey, const std::string& newkey);
void makeJob(const std::string& key);
// this struct kind of got sloppy. Need to clean it at some point.
struct PendingOps
{
PendingOps(int flags);
~PendingOps();
int opFlags;
int waiters;
bool finished;
boost::condition condvar;
void wait(boost::mutex*);
void notify();
};
struct Job : public ThreadPool::Job
{
virtual ~Job(){};
Job(Synchronizer* s, std::list<std::string>::iterator i);
void operator()();
Synchronizer* sync;
std::list<std::string>::iterator it;
};
uint maxUploads;
boost::scoped_ptr<ThreadPool> threadPool;
std::map<std::string, boost::shared_ptr<PendingOps> > pendingOps;
std::map<std::string, boost::shared_ptr<PendingOps> > opsInProgress;
// this is a bit of a kludge to handle objects being renamed. Jobs can be issued
// against name1, but when the job starts running, the target may be name2.
// some consolidation should be possible between this and the two maps above, tbd.
// in general the code got kludgier b/c of renaming, needs a cleanup pass.
std::list<std::string> objNames;
// this thread will start jobs for entries in pendingOps every 10 seconds
bool die;
boost::thread syncThread;
const boost::chrono::seconds syncInterval = boost::chrono::seconds(10);
void periodicSync();
std::map<boost::filesystem::path, size_t> uncommittedJournalSize;
size_t journalSizeThreshold;
bool blockNewJobs;
void syncNow(const boost::filesystem::path& prefix); // a synchronous version of forceFlush()
// some KPIs
size_t numBytesRead, numBytesWritten, numBytesUploaded, numBytesDownloaded, flushesTriggeredBySize,
flushesTriggeredByTimer, journalsMerged, objectsSyncedWithNoJournal, bytesReadBySync,
bytesReadBySyncWithJournal;
ssize_t mergeDiff;
SMLogging* logger;
Cache* cache;
Replicator* replicator;
IOCoordinator* ioc;
CloudStorage* cs;
boost::filesystem::path cachePath;
boost::filesystem::path journalPath;
boost::mutex mutex;
};
}
} // namespace storagemanager

230
storage-manager/src/ThreadPool.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "ThreadPool.h"
#include <iostream>
@ -23,165 +22,168 @@ using namespace std;
namespace storagemanager
{
ThreadPool::ThreadPool() : maxThreads(1000), die(false), processQueueOnExit(true), threadsWaiting(0)
{
// Using this ctor effectively limits the # of threads here to the natural limit of the
// context it's used in. In the CRP class for example, the # of active threads would be
// naturally limited by the # of concurrent operations.
logger = SMLogging::get();
pruner = boost::thread([this] { this->prune(); } );
// Using this ctor effectively limits the # of threads here to the natural limit of the
// context it's used in. In the CRP class for example, the # of active threads would be
// naturally limited by the # of concurrent operations.
logger = SMLogging::get();
pruner = boost::thread([this] { this->prune(); });
}
ThreadPool::ThreadPool(uint num_threads, bool _processQueueOnExit) : maxThreads(num_threads), die(false),
processQueueOnExit(_processQueueOnExit), threadsWaiting(0)
ThreadPool::ThreadPool(uint num_threads, bool _processQueueOnExit)
: maxThreads(num_threads), die(false), processQueueOnExit(_processQueueOnExit), threadsWaiting(0)
{
logger = SMLogging::get();
pruner = boost::thread([this] { this->prune(); } );
logger = SMLogging::get();
pruner = boost::thread([this] { this->prune(); });
}
ThreadPool::~ThreadPool()
{
boost::unique_lock<boost::mutex> s(mutex);
die = true;
if (!processQueueOnExit)
jobs.clear();
jobAvailable.notify_all();
s.unlock();
threads.join_all();
pruner.interrupt();
pruner.join();
boost::unique_lock<boost::mutex> s(mutex);
die = true;
if (!processQueueOnExit)
jobs.clear();
jobAvailable.notify_all();
s.unlock();
threads.join_all();
pruner.interrupt();
pruner.join();
}
void ThreadPool::setName(const string &_name)
void ThreadPool::setName(const string& _name)
{
name = _name;
name = _name;
}
void ThreadPool::addJob(const boost::shared_ptr<Job> &j)
void ThreadPool::addJob(const boost::shared_ptr<Job>& j)
{
boost::unique_lock<boost::mutex> s(mutex);
if (die)
return;
jobs.push_back(j);
// Start another thread if necessary
if (threadsWaiting == 0 && (threads.size() - pruneable.size()) < maxThreads) {
boost::thread *thread = threads.create_thread([this] { this->processingLoop(); });
s_threads.insert(thread);
//if (name == "Downloader")
// cout << "Threadpool (" << name << "): created a new thread. live threads = " <<
// threads.size() - pruneable.size() << " job count = " << jobs.size() << endl;
}
else
jobAvailable.notify_one();
boost::unique_lock<boost::mutex> s(mutex);
if (die)
return;
jobs.push_back(j);
// Start another thread if necessary
if (threadsWaiting == 0 && (threads.size() - pruneable.size()) < maxThreads)
{
boost::thread* thread = threads.create_thread([this] { this->processingLoop(); });
s_threads.insert(thread);
// if (name == "Downloader")
// cout << "Threadpool (" << name << "): created a new thread. live threads = " <<
// threads.size() - pruneable.size() << " job count = " << jobs.size() << endl;
}
else
jobAvailable.notify_one();
}
void ThreadPool::prune()
{
set<ID_Thread>::iterator it;
boost::unique_lock<boost::mutex> s(mutex);
while (1)
set<ID_Thread>::iterator it;
boost::unique_lock<boost::mutex> s(mutex);
while (1)
{
while (pruneable.empty() && !die)
somethingToPrune.wait(s);
if (die)
break;
for (auto& id : pruneable)
{
while (pruneable.empty() && !die)
somethingToPrune.wait(s);
if (die)
break;
for (auto &id : pruneable)
{
it = s_threads.find(id);
assert(it != s_threads.end());
it->thrd->join();
threads.remove_thread(it->thrd);
s_threads.erase(it);
}
pruneable.clear();
it = s_threads.find(id);
assert(it != s_threads.end());
it->thrd->join();
threads.remove_thread(it->thrd);
s_threads.erase(it);
}
pruneable.clear();
}
}
void ThreadPool::setMaxThreads(uint newMax)
{
boost::unique_lock<boost::mutex> s(mutex);
maxThreads = newMax;
boost::unique_lock<boost::mutex> s(mutex);
maxThreads = newMax;
}
int ThreadPool::currentQueueSize() const
{
boost::unique_lock<boost::mutex> s(mutex);
return jobs.size();
boost::unique_lock<boost::mutex> s(mutex);
return jobs.size();
}
void ThreadPool::processingLoop()
{
boost::unique_lock<boost::mutex> s(mutex);
boost::unique_lock<boost::mutex> s(mutex);
try
{
_processingLoop(s);
}
catch (...)
{
}
// _processingLoop returns with the lock held
pruneable.push_back(boost::this_thread::get_id());
somethingToPrune.notify_one();
// if (name == "Downloader")
// cout << "threadpool " << name << " thread exiting. jobs = " << jobs.size() <<
// " live thread count = " << threads.size() - pruneable.size() << endl;
}
void ThreadPool::_processingLoop(boost::unique_lock<boost::mutex>& s)
{
// cout << "Thread started" << endl;
while (threads.size() - pruneable.size() <=
maxThreads) // this is the scale-down mechanism when maxThreads is changed
{
while (jobs.empty() && !die)
{
threadsWaiting++;
bool timedout = !jobAvailable.timed_wait<>(s, idleThreadTimeout);
threadsWaiting--;
if (timedout && jobs.empty())
{
// cout << "Thread exiting b/c of timeout" << endl;
return;
}
}
if (jobs.empty())
{ // die was set, and there are no jobs left to process
// cout << "Thread exiting b/c of die (?)" << endl;
return;
}
boost::shared_ptr<Job> job = jobs.front();
jobs.pop_front();
s.unlock();
try
{
_processingLoop(s);
(*job)();
}
catch (...)
{}
// _processingLoop returns with the lock held
pruneable.push_back(boost::this_thread::get_id());
somethingToPrune.notify_one();
//if (name == "Downloader")
// cout << "threadpool " << name << " thread exiting. jobs = " << jobs.size() <<
// " live thread count = " << threads.size() - pruneable.size() << endl;
}
void ThreadPool::_processingLoop(boost::unique_lock<boost::mutex> &s)
{
//cout << "Thread started" << endl;
while (threads.size() - pruneable.size() <= maxThreads) // this is the scale-down mechanism when maxThreads is changed
catch (exception& e)
{
while (jobs.empty() && !die)
{
threadsWaiting++;
bool timedout = !jobAvailable.timed_wait<>(s, idleThreadTimeout);
threadsWaiting--;
if (timedout && jobs.empty())
{
//cout << "Thread exiting b/c of timeout" << endl;
return;
}
}
if (jobs.empty()) { // die was set, and there are no jobs left to process
//cout << "Thread exiting b/c of die (?)" << endl;
return;
}
boost::shared_ptr<Job> job = jobs.front();
jobs.pop_front();
s.unlock();
try
{
(*job)();
}
catch (exception &e)
{
logger->log(LOG_CRIT, "ThreadPool (%s): caught '%s' from a job", name.c_str(), e.what());
}
s.lock();
logger->log(LOG_CRIT, "ThreadPool (%s): caught '%s' from a job", name.c_str(), e.what());
}
//cout << "Thread exiting b/c of pruning logic" << endl;
s.lock();
}
// cout << "Thread exiting b/c of pruning logic" << endl;
}
inline bool ThreadPool::id_compare::operator()(const ID_Thread &t1, const ID_Thread &t2) const
inline bool ThreadPool::id_compare::operator()(const ID_Thread& t1, const ID_Thread& t2) const
{
return t1.id < t2.id;
return t1.id < t2.id;
}
ThreadPool::ID_Thread::ID_Thread(boost::thread::id &i): id(i)
ThreadPool::ID_Thread::ID_Thread(boost::thread::id& i) : id(i)
{
}
ThreadPool::ID_Thread::ID_Thread(boost::thread *t): id(t->get_id()), thrd(t)
ThreadPool::ID_Thread::ID_Thread(boost::thread* t) : id(t->get_id()), thrd(t)
{
}
}
} // namespace storagemanager

115
storage-manager/src/ThreadPool.h
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include <deque>
@ -27,66 +26,64 @@
namespace storagemanager
{
class ThreadPool : public boost::noncopyable
{
public:
ThreadPool(); // this ctor uses an 'unbounded' # of threads; relies on the context for a natural max,
ThreadPool(uint num_threads, bool processQueueOnExit = false); // this ctor lets caller specify a limit.
virtual ~ThreadPool();
// addJob doesn't block
class Job
{
public:
virtual ~Job(){};
virtual void operator()() = 0;
};
void addJob(const boost::shared_ptr<Job> &j);
void setMaxThreads(uint newMax);
int currentQueueSize() const;
void setName(const std::string &); // set the name of this threadpool (for debugging)
public:
ThreadPool(); // this ctor uses an 'unbounded' # of threads; relies on the context for a natural max,
ThreadPool(uint num_threads, bool processQueueOnExit = false); // this ctor lets caller specify a limit.
virtual ~ThreadPool();
private:
void processingLoop(); // the fcn run by each thread
void _processingLoop(boost::unique_lock<boost::mutex> &); // processingLoop() wraps _processingLoop() with thread management stuff.
SMLogging *logger;
uint maxThreads;
bool die;
bool processQueueOnExit;
int threadsWaiting;
std::string name;
boost::thread_group threads;
// the set s_threads below is intended to make pruning idle threads efficient.
// there should be a cleaner way to do it.
struct ID_Thread
{
ID_Thread(boost::thread::id &);
ID_Thread(boost::thread *);
boost::thread::id id;
boost::thread *thrd;
};
struct id_compare
{
bool operator()(const ID_Thread &, const ID_Thread &) const;
};
std::set<ID_Thread, id_compare> s_threads;
boost::condition jobAvailable;
std::deque<boost::shared_ptr<Job> > jobs;
mutable boost::mutex mutex;
const boost::posix_time::time_duration idleThreadTimeout = boost::posix_time::seconds(60);
boost::thread pruner;
boost::condition somethingToPrune;
std::vector<boost::thread::id> pruneable; // when a thread is about to return it puts its id here
void prune();
// addJob doesn't block
class Job
{
public:
virtual ~Job(){};
virtual void operator()() = 0;
};
void addJob(const boost::shared_ptr<Job>& j);
void setMaxThreads(uint newMax);
int currentQueueSize() const;
void setName(const std::string&); // set the name of this threadpool (for debugging)
private:
void processingLoop(); // the fcn run by each thread
void _processingLoop(boost::unique_lock<boost::mutex>&); // processingLoop() wraps _processingLoop() with
// thread management stuff.
SMLogging* logger;
uint maxThreads;
bool die;
bool processQueueOnExit;
int threadsWaiting;
std::string name;
boost::thread_group threads;
// the set s_threads below is intended to make pruning idle threads efficient.
// there should be a cleaner way to do it.
struct ID_Thread
{
ID_Thread(boost::thread::id&);
ID_Thread(boost::thread*);
boost::thread::id id;
boost::thread* thrd;
};
struct id_compare
{
bool operator()(const ID_Thread&, const ID_Thread&) const;
};
std::set<ID_Thread, id_compare> s_threads;
boost::condition jobAvailable;
std::deque<boost::shared_ptr<Job> > jobs;
mutable boost::mutex mutex;
const boost::posix_time::time_duration idleThreadTimeout = boost::posix_time::seconds(60);
boost::thread pruner;
boost::condition somethingToPrune;
std::vector<boost::thread::id> pruneable; // when a thread is about to return it puts its id here
void prune();
};
}
} // namespace storagemanager

89
storage-manager/src/TruncateTask.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "TruncateTask.h"
#include <errno.h>
#include "messageFormat.h"
@ -25,7 +24,6 @@ using namespace std;
namespace storagemanager
{
TruncateTask::TruncateTask(int sock, uint len) : PosixTask(sock, len)
{
}
@ -35,51 +33,52 @@ TruncateTask::~TruncateTask()
}
#define check_error(msg, ret) \
if (success<0) \
{ \
handleError(msg, errno); \
return ret; \
}
if (success < 0) \
{ \
handleError(msg, errno); \
return ret; \
}
bool TruncateTask::run()
{
SMLogging* logger = SMLogging::get();
int success;
uint8_t buf[1024] = {0};
if (getLength() > 1023) {
handleError("TruncateTask read", ENAMETOOLONG);
return false;
}
success = read(buf, getLength());
check_error("TruncateTask read", false);
truncate_cmd *cmd = (truncate_cmd *) buf;
#ifdef SM_TRACE
logger->log(LOG_DEBUG,"truncate %s newlength %i.",cmd->filename,cmd->length);
#endif
int err;
try
{
err = ioc->truncate(cmd->filename, cmd->length);
}
catch (exception &e)
{
logger->log(LOG_ERR, "TruncateTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err)
{
handleError("TruncateTask truncate", errno);
return true;
}
sm_response *resp = (sm_response *) buf;
resp->returnCode = 0;
return write(*resp, 0);
SMLogging* logger = SMLogging::get();
int success;
uint8_t buf[1024] = {0};
if (getLength() > 1023)
{
handleError("TruncateTask read", ENAMETOOLONG);
return false;
}
success = read(buf, getLength());
check_error("TruncateTask read", false);
truncate_cmd* cmd = (truncate_cmd*)buf;
#ifdef SM_TRACE
logger->log(LOG_DEBUG, "truncate %s newlength %i.", cmd->filename, cmd->length);
#endif
int err;
try
{
err = ioc->truncate(cmd->filename, cmd->length);
}
catch (exception& e)
{
logger->log(LOG_ERR, "TruncateTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err)
{
handleError("TruncateTask truncate", errno);
return true;
}
sm_response* resp = (sm_response*)buf;
resp->returnCode = 0;
return write(*resp, 0);
}
}
} // namespace storagemanager

21
storage-manager/src/TruncateTask.h
View File

@ -15,25 +15,22 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "PosixTask.h"
namespace storagemanager
{
class TruncateTask : public PosixTask
{
public:
TruncateTask(int sock, uint length);
virtual ~TruncateTask();
bool run();
private:
TruncateTask();
public:
TruncateTask(int sock, uint length);
virtual ~TruncateTask();
bool run();
private:
TruncateTask();
};
}
} // namespace storagemanager

92
storage-manager/src/UnlinkTask.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "UnlinkTask.h"
#include <errno.h>
#include "messageFormat.h"
@ -25,7 +24,6 @@ using namespace std;
namespace storagemanager
{
UnlinkTask::UnlinkTask(int sock, uint len) : PosixTask(sock, len)
{
}
@ -35,53 +33,53 @@ UnlinkTask::~UnlinkTask()
}
#define check_error(msg, ret) \
if (success<0) \
{ \
handleError(msg, errno); \
return ret; \
}
if (success < 0) \
{ \
handleError(msg, errno); \
return ret; \
}
bool UnlinkTask::run()
{
SMLogging* logger = SMLogging::get();
int success;
uint8_t buf[1024] = {0};
if (getLength() > 1023) {
handleError("UnlinkTask read", ENAMETOOLONG);
return true;
}
success = read(buf, getLength());
check_error("UnlinkTask read", false);
unlink_cmd *cmd = (unlink_cmd *) buf;
#ifdef SM_TRACE
logger->log(LOG_DEBUG,"unlink %s.",cmd->filename);
#endif
int err;
try
{
err = ioc->unlink(cmd->filename);
}
catch (exception &e)
{
logger->log(LOG_ERR, "UnlinkTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err)
{
handleError("UnlinkTask unlink", errno);
return true;
}
sm_response *resp = (sm_response *) buf;
resp->returnCode = 0;
return write(*resp, 0);
SMLogging* logger = SMLogging::get();
int success;
uint8_t buf[1024] = {0};
if (getLength() > 1023)
{
handleError("UnlinkTask read", ENAMETOOLONG);
return true;
}
success = read(buf, getLength());
check_error("UnlinkTask read", false);
unlink_cmd* cmd = (unlink_cmd*)buf;
#ifdef SM_TRACE
logger->log(LOG_DEBUG, "unlink %s.", cmd->filename);
#endif
int err;
try
{
err = ioc->unlink(cmd->filename);
}
catch (exception& e)
{
logger->log(LOG_ERR, "UnlinkTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err)
{
handleError("UnlinkTask unlink", errno);
return true;
}
sm_response* resp = (sm_response*)buf;
resp->returnCode = 0;
return write(*resp, 0);
}
}
} // namespace storagemanager

21
storage-manager/src/UnlinkTask.h
View File

@ -15,25 +15,22 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "PosixTask.h"
namespace storagemanager
{
class UnlinkTask : public PosixTask
{
public:
UnlinkTask(int sock, uint length);
virtual ~UnlinkTask();
bool run();
private:
UnlinkTask();
public:
UnlinkTask(int sock, uint length);
virtual ~UnlinkTask();
bool run();
private:
UnlinkTask();
};
}
} // namespace storagemanager

113
storage-manager/src/Utilities.cpp
View File

@ -20,8 +20,7 @@
namespace storagemanager
{
ScopedFileLock::ScopedFileLock(IOCoordinator *i, const std::string &k) : ioc(i), locked(false), key(k)
ScopedFileLock::ScopedFileLock(IOCoordinator* i, const std::string& k) : ioc(i), locked(false), key(k)
{
}
@ -29,91 +28,97 @@ ScopedFileLock::~ScopedFileLock()
{
}
ScopedReadLock::ScopedReadLock(IOCoordinator *i, const std::string &k) : ScopedFileLock(i, k)
ScopedReadLock::ScopedReadLock(IOCoordinator* i, const std::string& k) : ScopedFileLock(i, k)
{
lock();
lock();
}
ScopedReadLock::~ScopedReadLock()
{
unlock();
unlock();
}
void ScopedReadLock::lock()
{
assert(!locked);
ioc->readLock(key);
locked = true;
assert(!locked);
ioc->readLock(key);
locked = true;
}
void ScopedReadLock::unlock()
{
if (locked)
{
ioc->readUnlock(key);
locked = false;
}
if (locked)
{
ioc->readUnlock(key);
locked = false;
}
}
ScopedWriteLock::ScopedWriteLock(IOCoordinator *i, const std::string &k) : ScopedFileLock(i, k)
ScopedWriteLock::ScopedWriteLock(IOCoordinator* i, const std::string& k) : ScopedFileLock(i, k)
{
lock();
lock();
}
ScopedWriteLock::~ScopedWriteLock()
{
unlock();
unlock();
}
void ScopedWriteLock::lock()
{
assert(!locked);
ioc->writeLock(key);
locked = true;
assert(!locked);
ioc->writeLock(key);
locked = true;
}
void ScopedWriteLock::unlock()
{
if (locked)
{
ioc->writeUnlock(key);
locked = false;
}
if (locked)
{
ioc->writeUnlock(key);
locked = false;
}
}
ScopedCloser::ScopedCloser() : fd(-1) { }
ScopedCloser::ScopedCloser(int f) : fd(f) { assert(f != -1); }
ScopedCloser::~ScopedCloser() {
if (fd < 0)
return;
int s_errno = errno;
::close(fd);
errno = s_errno;
}
SharedCloser::SharedCloser(int f)
{
block = new CtrlBlock();
assert(f >= 0);
block->fd = f;
block->refCount = 1;
}
SharedCloser::SharedCloser(const SharedCloser &s) : block(s.block)
ScopedCloser::ScopedCloser() : fd(-1)
{
block->refCount++;
}
ScopedCloser::ScopedCloser(int f) : fd(f)
{
assert(f != -1);
}
ScopedCloser::~ScopedCloser()
{
if (fd < 0)
return;
int s_errno = errno;
::close(fd);
errno = s_errno;
}
SharedCloser::SharedCloser(int f)
{
block = new CtrlBlock();
assert(f >= 0);
block->fd = f;
block->refCount = 1;
}
SharedCloser::SharedCloser(const SharedCloser& s) : block(s.block)
{
block->refCount++;
}
SharedCloser::~SharedCloser()
{
block->refCount--;
if (block->refCount == 0)
{
int s_errno = errno;
::close(block->fd);
delete block;
errno = s_errno;
}
}
block->refCount--;
if (block->refCount == 0)
{
int s_errno = errno;
::close(block->fd);
delete block;
errno = s_errno;
}
}
} // namespace storagemanager

73
storage-manager/src/Utilities.h
View File

@ -19,10 +19,8 @@
#include <string>
namespace storagemanager
{
class IOCoordinator;
// a few utility classes we've coded here and there, now de-duped and centralized.
@ -30,57 +28,56 @@ class IOCoordinator;
struct ScopedFileLock
{
ScopedFileLock(IOCoordinator *i, const std::string &k);
virtual ~ScopedFileLock();
virtual void lock() = 0;
virtual void unlock() = 0;
IOCoordinator *ioc;
bool locked;
const std::string key;
ScopedFileLock(IOCoordinator* i, const std::string& k);
virtual ~ScopedFileLock();
virtual void lock() = 0;
virtual void unlock() = 0;
IOCoordinator* ioc;
bool locked;
const std::string key;
};
struct ScopedReadLock : public ScopedFileLock
{
ScopedReadLock(IOCoordinator *i, const std::string &k);
virtual ~ScopedReadLock();
void lock();
void unlock();
ScopedReadLock(IOCoordinator* i, const std::string& k);
virtual ~ScopedReadLock();
void lock();
void unlock();
};
struct ScopedWriteLock : public ScopedFileLock
{
ScopedWriteLock(IOCoordinator *i, const std::string &k);
virtual ~ScopedWriteLock();
void lock();
void unlock();
ScopedWriteLock(IOCoordinator* i, const std::string& k);
virtual ~ScopedWriteLock();
void lock();
void unlock();
};
struct ScopedCloser
{
ScopedCloser();
ScopedCloser(int f);
~ScopedCloser();
int fd;
ScopedCloser();
ScopedCloser(int f);
~ScopedCloser();
int fd;
};
class SharedCloser
{
public:
SharedCloser(int f);
SharedCloser(const SharedCloser &);
~SharedCloser();
private:
struct CtrlBlock
{
int fd;
uint refCount;
};
CtrlBlock *block;
public:
SharedCloser(int f);
SharedCloser(const SharedCloser&);
~SharedCloser();
private:
struct CtrlBlock
{
int fd;
uint refCount;
};
CtrlBlock* block;
};
}
} // namespace storagemanager

146
storage-manager/src/WriteTask.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include "WriteTask.h"
#include "messageFormat.h"
#include "IOCoordinator.h"
@ -26,7 +25,6 @@ using namespace std;
namespace storagemanager
{
WriteTask::WriteTask(int sock, uint len) : PosixTask(sock, len)
{
}
@ -36,85 +34,85 @@ WriteTask::~WriteTask()
}
#define check_error(msg, ret) \
if (success<0) \
{ \
handleError(msg, errno); \
return ret; \
}
if (success < 0) \
{ \
handleError(msg, errno); \
return ret; \
}
#define min(x, y) (x < y ? x : y)
bool WriteTask::run()
{
SMLogging* logger = SMLogging::get();
int success;
uint8_t cmdbuf[1024] = {0};
success = read(cmdbuf, sizeof(write_cmd));
check_error("WriteTask read", false);
write_cmd *cmd = (write_cmd *) cmdbuf;
if (cmd->flen > 1023 - sizeof(*cmd))
{
handleError("WriteTask", ENAMETOOLONG);
return true;
}
success = read(&cmdbuf[sizeof(*cmd)], cmd->flen);
check_error("WriteTask read", false);
#ifdef SM_TRACE
logger->log(LOG_DEBUG,"write filename %s offset %i count %i.",cmd->filename,cmd->offset,cmd->count);
#endif
ssize_t readCount = 0, writeCount = 0;
vector<uint8_t> databuf;
uint bufsize = min(100 << 20, cmd->count); // 100 MB
//uint bufsize = cmd->count;
databuf.resize(bufsize);
SMLogging* logger = SMLogging::get();
int success;
uint8_t cmdbuf[1024] = {0};
while (readCount < cmd->count)
success = read(cmdbuf, sizeof(write_cmd));
check_error("WriteTask read", false);
write_cmd* cmd = (write_cmd*)cmdbuf;
if (cmd->flen > 1023 - sizeof(*cmd))
{
handleError("WriteTask", ENAMETOOLONG);
return true;
}
success = read(&cmdbuf[sizeof(*cmd)], cmd->flen);
check_error("WriteTask read", false);
#ifdef SM_TRACE
logger->log(LOG_DEBUG, "write filename %s offset %i count %i.", cmd->filename, cmd->offset, cmd->count);
#endif
ssize_t readCount = 0, writeCount = 0;
vector<uint8_t> databuf;
uint bufsize = min(100 << 20, cmd->count); // 100 MB
// uint bufsize = cmd->count;
databuf.resize(bufsize);
while (readCount < cmd->count)
{
uint toRead = min(static_cast<uint>(cmd->count - readCount), bufsize);
success = read(&databuf[0], toRead);
check_error("WriteTask read data", false);
if (success == 0)
break;
readCount += success;
uint writePos = 0;
ssize_t err;
while (writeCount < readCount)
{
uint toRead = min(static_cast<uint>(cmd->count - readCount), bufsize);
success = read(&databuf[0], toRead);
check_error("WriteTask read data", false);
if (success==0)
break;
readCount += success;
uint writePos = 0;
ssize_t err;
while (writeCount < readCount)
{
try
{
err = ioc->write(cmd->filename, &databuf[writePos], cmd->offset + writeCount, success - writePos);
}
catch (exception &e)
{
logger->log(LOG_ERR, "WriteTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err <= 0)
break;
writeCount += err;
writePos += err;
}
if (writeCount != readCount)
break;
try
{
err = ioc->write(cmd->filename, &databuf[writePos], cmd->offset + writeCount, success - writePos);
}
catch (exception& e)
{
logger->log(LOG_ERR, "WriteTask: caught '%s'", e.what());
errno = EIO;
err = -1;
}
if (err <= 0)
break;
writeCount += err;
writePos += err;
}
uint8_t respbuf[sizeof(sm_response) + 4];
sm_response *resp = (sm_response *) respbuf;
uint payloadLen = 0;
if (cmd->count != 0 && writeCount == 0)
{
payloadLen = 4;
resp->returnCode = -1;
*((int *) &resp[1]) = errno;
}
else
resp->returnCode = writeCount;
return write(*resp, payloadLen);
if (writeCount != readCount)
break;
}
uint8_t respbuf[sizeof(sm_response) + 4];
sm_response* resp = (sm_response*)respbuf;
uint payloadLen = 0;
if (cmd->count != 0 && writeCount == 0)
{
payloadLen = 4;
resp->returnCode = -1;
*((int*)&resp[1]) = errno;
}
else
resp->returnCode = writeCount;
return write(*resp, payloadLen);
}
}
} // namespace storagemanager

21
storage-manager/src/WriteTask.h
View File

@ -15,25 +15,22 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#pragma once
#include "PosixTask.h"
namespace storagemanager
{
class WriteTask : public PosixTask
{
public:
WriteTask(int sock, uint length);
virtual ~WriteTask();
bool run();
private:
WriteTask();
public:
WriteTask(int sock, uint length);
virtual ~WriteTask();
bool run();
private:
WriteTask();
};
}
} // namespace storagemanager

201
storage-manager/src/main.cpp
View File

@ -15,7 +15,6 @@
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
#include <unistd.h>
#include <string>
#include <iostream>
@ -39,160 +38,150 @@ using namespace std;
using namespace storagemanager;
class Opt
{
public:
const char *m_progname;
public:
const char* m_progname;
bool m_fg;
Opt(int argc, char **argv)
:m_progname(argv[0]),
m_fg(argc >= 2 && string(argv[1]) == "fg")
{ }
Opt(int argc, char** argv) : m_progname(argv[0]), m_fg(argc >= 2 && string(argv[1]) == "fg")
{
}
};
class ServiceStorageManager: public Service, public Opt
class ServiceStorageManager : public Service, public Opt
{
protected:
void setupChildSignalHandlers();
protected:
void setupChildSignalHandlers();
public:
ServiceStorageManager(const Opt &opt)
:Service("StorageManager"), Opt(opt)
{ }
void LogErrno() override
{
SMLogging::get()->log(LOG_ERR, "%s", strerror(errno));
}
void ParentLogChildMessage(const std::string &str) override
{
SMLogging::get()->log(LOG_INFO, "%.*s", (int) str.length(), str.data());
}
int Child() override;
int Run()
{
return m_fg ? Child() : RunForking();
}
public:
ServiceStorageManager(const Opt& opt) : Service("StorageManager"), Opt(opt)
{
}
void LogErrno() override
{
SMLogging::get()->log(LOG_ERR, "%s", strerror(errno));
}
void ParentLogChildMessage(const std::string& str) override
{
SMLogging::get()->log(LOG_INFO, "%.*s", (int)str.length(), str.data());
}
int Child() override;
int Run()
{
return m_fg ? Child() : RunForking();
}
};
bool signalCaught = false;
void printCacheUsage(int sig)
{
Cache::get()->validateCacheSize();
cout << "Current cache size = " << Cache::get()->getCurrentCacheSize() << endl;
cout << "Cache element count = " << Cache::get()->getCurrentCacheElementCount() << endl;
Cache::get()->validateCacheSize();
cout << "Current cache size = " << Cache::get()->getCurrentCacheSize() << endl;
cout << "Cache element count = " << Cache::get()->getCurrentCacheElementCount() << endl;
}
void printKPIs(int sig)
{
IOCoordinator::get()->printKPIs();
Cache::get()->printKPIs();
Synchronizer::get()->printKPIs();
CloudStorage::get()->printKPIs();
Replicator::get()->printKPIs();
IOCoordinator::get()->printKPIs();
Cache::get()->printKPIs();
Synchronizer::get()->printKPIs();
CloudStorage::get()->printKPIs();
Replicator::get()->printKPIs();
}
void shutdownSM(int sig)
{
if (!signalCaught)
{
(SessionManager::get())->shutdownSM(sig);
}
signalCaught = true;
if (!signalCaught)
{
(SessionManager::get())->shutdownSM(sig);
}
signalCaught = true;
}
void coreSM(int sig)
{
if (!signalCaught)
{
fatalHandler(sig);
}
signalCaught = true;
if (!signalCaught)
{
fatalHandler(sig);
}
signalCaught = true;
}
void ServiceStorageManager::setupChildSignalHandlers()
{
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
struct sigaction sa;
memset(&sa, 0, sizeof(sa));
std::vector<int> shutdownSignals{ SIGALRM, SIGHUP, SIGINT, SIGPOLL,
SIGPROF, SIGPWR, SIGTERM, SIGVTALRM};
std::vector<int> shutdownSignals{SIGALRM, SIGHUP, SIGINT, SIGPOLL, SIGPROF, SIGPWR, SIGTERM, SIGVTALRM};
std::vector<int> coreSignals{SIGABRT, SIGBUS, SIGFPE, SIGILL,
SIGQUIT, SIGSEGV, SIGSYS, SIGTRAP,
SIGXCPU, SIGXFSZ};
std::vector<int> coreSignals{SIGABRT, SIGBUS, SIGFPE, SIGILL, SIGQUIT,
SIGSEGV, SIGSYS, SIGTRAP, SIGXCPU, SIGXFSZ};
sa.sa_handler = shutdownSM;
for (int sig : shutdownSignals)
sigaction(sig, &sa, NULL);
sa.sa_handler = coreSM;
for (int sig : coreSignals)
sigaction(sig, &sa, NULL);
sa.sa_handler = shutdownSM;
for (int sig : shutdownSignals)
sigaction(sig, &sa, NULL);
sa.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &sa, NULL);
sa.sa_handler = printCacheUsage;
sigaction(SIGUSR1, &sa, NULL);
sa.sa_handler = printKPIs;
sigaction(SIGUSR2, &sa, NULL);
sa.sa_handler = coreSM;
for (int sig : coreSignals)
sigaction(sig, &sa, NULL);
sa.sa_handler = SIG_IGN;
sigaction(SIGPIPE, &sa, NULL);
sa.sa_handler = printCacheUsage;
sigaction(SIGUSR1, &sa, NULL);
sa.sa_handler = printKPIs;
sigaction(SIGUSR2, &sa, NULL);
}
int ServiceStorageManager::Child()
{
SMLogging* logger = SMLogging::get();
IOCoordinator* ioc = NULL;
Cache* cache = NULL;
Synchronizer* sync = NULL;
Replicator* rep = NULL;
SMLogging* logger = SMLogging::get();
IOCoordinator* ioc = NULL;
Cache* cache = NULL;
Synchronizer* sync = NULL;
Replicator* rep = NULL;
/* Instantiate objects to have them verify config settings before continuing */
try
{
ioc = IOCoordinator::get();
cache = Cache::get();
sync = Synchronizer::get();
rep = Replicator::get();
}
catch (exception& e)
{
logger->log(LOG_INFO, "StorageManager init FAIL: %s", e.what());
return -1;
}
/* Instantiate objects to have them verify config settings before continuing */
try
{
ioc = IOCoordinator::get();
cache = Cache::get();
sync = Synchronizer::get();
rep = Replicator::get();
}
catch (exception &e)
{
logger->log(LOG_INFO, "StorageManager init FAIL: %s", e.what());
return -1;
}
setupChildSignalHandlers();
setupChildSignalHandlers();
int ret = 0;
int ret = 0;
logger->log(LOG_NOTICE, "StorageManager started.");
logger->log(LOG_NOTICE,"StorageManager started.");
SessionManager* sm = SessionManager::get();
SessionManager* sm = SessionManager::get();
NotifyServiceStarted();
NotifyServiceStarted();
ret = sm->start();
ret = sm->start();
cache->shutdown();
cache->shutdown();
delete sync;
delete cache;
delete ioc;
delete rep;
logger->log(LOG_INFO,"StorageManager Shutdown Complete.");
delete sync;
delete cache;
delete ioc;
delete rep;
logger->log(LOG_INFO, "StorageManager Shutdown Complete.");
return ret;
return ret;
}
int main(int argc, char** argv)
{
return ServiceStorageManager(Opt(argc, argv)).Run();
return ServiceStorageManager(Opt(argc, argv)).Run();
}

226
storage-manager/src/smcat.cpp
View File

@ -28,139 +28,143 @@
using namespace std;
using namespace storagemanager;
void usage(const char *progname)
void usage(const char* progname)
{
cerr << progname << " is 'cat' for files managed by StorageManager." << endl;
cerr << "Usage: " << progname << " file1 file2 ... fileN" << endl;
cerr << progname << " is 'cat' for files managed by StorageManager." << endl;
cerr << "Usage: " << progname << " file1 file2 ... fileN" << endl;
}
bool SMOnline()
{
struct sockaddr_un addr;
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null...
int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0);
int err = ::connect(clientSocket, (const struct sockaddr *) &addr, sizeof(addr));
if (err >= 0)
{
::close(clientSocket);
return true;
}
return false;
struct sockaddr_un addr;
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null...
int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0);
int err = ::connect(clientSocket, (const struct sockaddr*)&addr, sizeof(addr));
if (err >= 0)
{
::close(clientSocket);
return true;
}
return false;
}
void catFileOffline(const char *filename, int prefixlen)
void catFileOffline(const char* filename, int prefixlen)
{
uint8_t data[8192];
off_t offset = 0;
int read_err, write_err, count;
try
uint8_t data[8192];
off_t offset = 0;
int read_err, write_err, count;
try
{
boost::scoped_ptr<IOCoordinator> ioc(IOCoordinator::get());
do
{
boost::scoped_ptr<IOCoordinator> ioc(IOCoordinator::get());
count = 0;
read_err = ioc->read(filename, data, offset, 8192);
if (read_err < 0)
{
int l_errno = errno;
cerr << "Error reading " << &filename[prefixlen] << ": " << strerror_r(l_errno, (char*)data, 8192)
<< endl;
}
do {
count = 0;
read_err = ioc->read(filename, data, offset, 8192);
if (read_err < 0)
{
int l_errno = errno;
cerr << "Error reading " << &filename[prefixlen] << ": " << strerror_r(l_errno, (char *) data, 8192) << endl;
}
while (count < read_err)
{
write_err = write(STDOUT_FILENO, &data[count], read_err - count);
if (write_err < 0)
{
int l_errno = errno;
cerr << "Error writing to stdout: " << strerror_r(l_errno, (char *) data, 8192) << endl;
exit(1);
}
count += write_err;
}
offset += read_err;
} while (read_err > 0);
}
catch (exception &e)
{
cerr << "smcat catFileOffline FAIL: " << e.what() << endl;
}
}
void catFileOnline(const char *filename, int prefixlen)
{
uint8_t data[8192];
off_t offset = 0;
int read_err, write_err, count;
idbdatafile::SMDataFile df(filename, O_RDONLY, 0);
do {
count = 0;
read_err = df.read(data, 8192);
if (read_err < 0)
while (count < read_err)
{
write_err = write(STDOUT_FILENO, &data[count], read_err - count);
if (write_err < 0)
{
int l_errno = errno;
cerr << "Error reading " << &filename[prefixlen] << ": " << strerror_r(l_errno, (char *) data, 8192) << endl;
int l_errno = errno;
cerr << "Error writing to stdout: " << strerror_r(l_errno, (char*)data, 8192) << endl;
exit(1);
}
while (count < read_err)
{
write_err = write(STDOUT_FILENO, &data[count], read_err - count);
if (write_err < 0)
{
int l_errno = errno;
cerr << "Error writing to stdout: " << strerror_r(l_errno, (char *) data, 8192) << endl;
exit(1);
}
count += write_err;
}
offset += read_err;
count += write_err;
}
offset += read_err;
} while (read_err > 0);
}
catch (exception& e)
{
cerr << "smcat catFileOffline FAIL: " << e.what() << endl;
}
}
int makePathPrefix(char *target, int targetlen)
void catFileOnline(const char* filename, int prefixlen)
{
// MCOL-3438 -> add bogus directories to the front of each param
Config *config = Config::get();
int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth"));
target[0] = '/';
target[1] = 0;
int bufpos = 1;
for (int i = 0; i < prefixDepth; i++)
uint8_t data[8192];
off_t offset = 0;
int read_err, write_err, count;
idbdatafile::SMDataFile df(filename, O_RDONLY, 0);
do
{
count = 0;
read_err = df.read(data, 8192);
if (read_err < 0)
{
if (bufpos + 3 >= targetlen)
{
cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth";
exit(1);
}
memcpy(&target[bufpos], "x/\0", 3);
bufpos += 2;
int l_errno = errno;
cerr << "Error reading " << &filename[prefixlen] << ": " << strerror_r(l_errno, (char*)data, 8192)
<< endl;
}
return bufpos;
while (count < read_err)
{
write_err = write(STDOUT_FILENO, &data[count], read_err - count);
if (write_err < 0)
{
int l_errno = errno;
cerr << "Error writing to stdout: " << strerror_r(l_errno, (char*)data, 8192) << endl;
exit(1);
}
count += write_err;
}
offset += read_err;
} while (read_err > 0);
}
int main(int argc, char **argv)
int makePathPrefix(char* target, int targetlen)
{
if (argc < 2)
{
usage(argv[0]);
return 1;
}
char prefix[8192];
int prefixlen = makePathPrefix(prefix, 8192);
// MCOL-3438 -> add bogus directories to the front of each param
Config* config = Config::get();
int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth"));
target[0] = '/';
target[1] = 0;
int bufpos = 1;
for (int i = 1; i < argc; i++)
for (int i = 0; i < prefixDepth; i++)
{
if (bufpos + 3 >= targetlen)
{
strncat(&prefix[prefixlen], argv[i], 8192 - prefixlen);
if (SMOnline())
catFileOnline(prefix, prefixlen);
else
catFileOffline(prefix, prefixlen);
cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth";
exit(1);
}
return 0;
memcpy(&target[bufpos], "x/\0", 3);
bufpos += 2;
}
return bufpos;
}
int main(int argc, char** argv)
{
if (argc < 2)
{
usage(argv[0]);
return 1;
}
char prefix[8192];
int prefixlen = makePathPrefix(prefix, 8192);
for (int i = 1; i < argc; i++)
{
strncat(&prefix[prefixlen], argv[i], 8192 - prefixlen);
if (SMOnline())
catFileOnline(prefix, prefixlen);
else
catFileOffline(prefix, prefixlen);
}
return 0;
}

249
storage-manager/src/smls.cpp
View File

@ -34,155 +34,154 @@
using namespace std;
using namespace storagemanager;
void usage(const char *progname)
void usage(const char* progname)
{
cerr << progname << " is like 'ls -l' for files managed by StorageManager" << endl;
cerr << "Usage: " << progname << " directory" << endl;
cerr << progname << " is like 'ls -l' for files managed by StorageManager" << endl;
cerr << "Usage: " << progname << " directory" << endl;
}
bool SMOnline()
{
struct sockaddr_un addr;
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null...
int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0);
int err = ::connect(clientSocket, (const struct sockaddr *) &addr, sizeof(addr));
if (err >= 0)
{
::close(clientSocket);
return true;
}
return false;
struct sockaddr_un addr;
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null...
int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0);
int err = ::connect(clientSocket, (const struct sockaddr*)&addr, sizeof(addr));
if (err >= 0)
{
::close(clientSocket);
return true;
}
return false;
}
void lsOffline(const char *path)
void lsOffline(const char* path)
{
try
{
boost::scoped_ptr<IOCoordinator> ioc(IOCoordinator::get());
vector<string> listing;
try
{
boost::scoped_ptr<IOCoordinator> ioc(IOCoordinator::get());
vector<string> listing;
int err = ioc->listDirectory(path, &listing);
if (err)
exit(1);
struct stat _stat;
boost::filesystem::path base(path);
boost::filesystem::path p;
cout.fill(' ');
for (auto &entry : listing)
{
p = base / entry;
err = ioc->stat(p.string().c_str(), &_stat);
if (!err)
{
if (_stat.st_mode & S_IFDIR)
{
cout << "d";
cout.width(14);
}
else
cout.width(15);
struct tm *my_tm = localtime(&_stat.st_mtim.tv_sec);
char date[100];
strftime(date, 100, "%b %e %H:%M", my_tm);
cout << right << _stat.st_size << left << " " << date << left << " " << entry << endl;
}
else
{
cout.width(15);
cout << right << "error" << left << " " << entry << endl;
}
}
}
catch (exception &e)
{
cerr << "smls lsOffline FAIL: " << e.what() << endl;
}
}
void lsOnline(const char *path)
{
idbdatafile::SMFileSystem fs;
list<string> listing;
int err = fs.listDirectory(path, listing);
int err = ioc->listDirectory(path, &listing);
if (err)
exit(1);
exit(1);
struct stat _stat;
boost::filesystem::path base(path);
boost::filesystem::path p;
cout.fill(' ');
for (auto &entry : listing)
for (auto& entry : listing)
{
p = base / entry;
bool isDir = fs.isDir(p.string().c_str());
ssize_t size = fs.size(p.string().c_str());
idbdatafile::SMDataFile df(p.string().c_str(),O_RDONLY,1);
time_t mtime = df.mtime();
if (size >= 0)
p = base / entry;
err = ioc->stat(p.string().c_str(), &_stat);
if (!err)
{
if (_stat.st_mode & S_IFDIR)
{
if (isDir)
{
cout << "d";
cout.width(14);
}
else
cout.width(15);
struct tm *my_tm = localtime(&mtime);
char date[100];
strftime(date, 100, "%b %e %H:%M", my_tm);
cout << right << size << left << " " << date << left << " " << entry << endl;
cout << "d";
cout.width(14);
}
else
{
cout.width(15);
cout << right << "error" << left << " " << entry << endl;
}
cout.width(15);
struct tm* my_tm = localtime(&_stat.st_mtim.tv_sec);
char date[100];
strftime(date, 100, "%b %e %H:%M", my_tm);
cout << right << _stat.st_size << left << " " << date << left << " " << entry << endl;
}
else
{
cout.width(15);
cout << right << "error" << left << " " << entry << endl;
}
}
}
catch (exception& e)
{
cerr << "smls lsOffline FAIL: " << e.what() << endl;
}
}
int makePathPrefix(char *target, int targetlen)
void lsOnline(const char* path)
{
// MCOL-3438 -> add bogus directories to the front of each param
Config *config = Config::get();
int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth"));
target[0] = '/';
target[1] = 0;
int bufpos = 1;
for (int i = 0; i < prefixDepth; i++)
{
if (bufpos + 3 >= targetlen)
{
cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth";
exit(1);
}
memcpy(&target[bufpos], "x/\0", 3);
bufpos += 2;
}
return bufpos;
}
idbdatafile::SMFileSystem fs;
list<string> listing;
int main(int argc, char **argv)
{
if (argc != 2)
int err = fs.listDirectory(path, listing);
if (err)
exit(1);
boost::filesystem::path base(path);
boost::filesystem::path p;
cout.fill(' ');
for (auto& entry : listing)
{
p = base / entry;
bool isDir = fs.isDir(p.string().c_str());
ssize_t size = fs.size(p.string().c_str());
idbdatafile::SMDataFile df(p.string().c_str(), O_RDONLY, 1);
time_t mtime = df.mtime();
if (size >= 0)
{
usage(argv[0]);
return 1;
if (isDir)
{
cout << "d";
cout.width(14);
}
else
cout.width(15);
struct tm* my_tm = localtime(&mtime);
char date[100];
strftime(date, 100, "%b %e %H:%M", my_tm);
cout << right << size << left << " " << date << left << " " << entry << endl;
}
char prefix[8192];
int prefixlen = makePathPrefix(prefix, 8192);
if (SMOnline())
lsOnline(strncat(prefix, argv[1], 8192 - prefixlen));
else
lsOffline(strncat(prefix, argv[1], 8192 - prefixlen));
return 0;
{
cout.width(15);
cout << right << "error" << left << " " << entry << endl;
}
}
}
int makePathPrefix(char* target, int targetlen)
{
// MCOL-3438 -> add bogus directories to the front of each param
Config* config = Config::get();
int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth"));
target[0] = '/';
target[1] = 0;
int bufpos = 1;
for (int i = 0; i < prefixDepth; i++)
{
if (bufpos + 3 >= targetlen)
{
cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth";
exit(1);
}
memcpy(&target[bufpos], "x/\0", 3);
bufpos += 2;
}
return bufpos;
}
int main(int argc, char** argv)
{
if (argc != 2)
{
usage(argv[0]);
return 1;
}
char prefix[8192];
int prefixlen = makePathPrefix(prefix, 8192);
if (SMOnline())
lsOnline(strncat(prefix, argv[1], 8192 - prefixlen));
else
lsOffline(strncat(prefix, argv[1], 8192 - prefixlen));
return 0;
}

254
storage-manager/src/smput.cpp
View File

@ -30,158 +30,156 @@
using namespace std;
using namespace storagemanager;
void usage(const char *progname)
void usage(const char* progname)
{
cerr << progname << " reads from stdin and puts it in a file managed by StorageManager" << endl;
cerr << "Usage: " << progname << " output_file" << endl;
cerr << progname << " reads from stdin and puts it in a file managed by StorageManager" << endl;
cerr << "Usage: " << progname << " output_file" << endl;
}
bool SMOnline()
{
struct sockaddr_un addr;
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null...
int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0);
int err = ::connect(clientSocket, (const struct sockaddr *) &addr, sizeof(addr));
if (err >= 0)
{
::close(clientSocket);
return true;
}
return false;
struct sockaddr_un addr;
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null...
int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0);
int err = ::connect(clientSocket, (const struct sockaddr*)&addr, sizeof(addr));
if (err >= 0)
{
::close(clientSocket);
return true;
}
return false;
}
void putOffline(const char *fname, int prefixlen)
void putOffline(const char* fname, int prefixlen)
{
uint8_t data[8192];
int read_err, write_err;
ssize_t count, offset = 0;
try
uint8_t data[8192];
int read_err, write_err;
ssize_t count, offset = 0;
try
{
boost::scoped_ptr<IOCoordinator> ioc(IOCoordinator::get());
struct stat _stat;
read_err = ioc->open(fname, O_CREAT | O_TRUNC | O_WRONLY, &_stat);
if (read_err < 0)
{
boost::scoped_ptr<IOCoordinator> ioc(IOCoordinator::get());
struct stat _stat;
read_err = ioc->open(fname, O_CREAT | O_TRUNC | O_WRONLY, &_stat);
if (read_err < 0)
{
int l_errno = errno;
cerr << "Failed to open/create " << &fname[prefixlen] << ": " <<
strerror_r(l_errno, (char *) data, 8192) << endl;
exit(1);
}
do
{
read_err = ::read(STDIN_FILENO, data, 8192);
if (read_err < 0)
{
int l_errno = errno;
cerr << "Error reading stdin: " << strerror_r(l_errno, (char *) data, 8192) << endl;
exit(1);
}
count = 0;
while (count < read_err)
{
write_err = ioc->write(fname, &data[count], offset + count, read_err - count);
if (write_err < 0)
{
int l_errno = errno;
cerr << "Error writing to " << &fname[prefixlen] << ": " <<
strerror_r(l_errno, (char *) data, 8192) << endl;
exit(1);
}
count += write_err;
}
offset += read_err;
} while (read_err > 0);
int l_errno = errno;
cerr << "Failed to open/create " << &fname[prefixlen] << ": " << strerror_r(l_errno, (char*)data, 8192)
<< endl;
exit(1);
}
catch (exception &e)
{
cerr << "smput putOffline FAIL: " << e.what() << endl;
}
}
void putOnline(const char *fname, int prefixlen)
{
uint8_t data[8192];
int read_err, write_err;
ssize_t count;
idbdatafile::SMFileFactory ffactory;
boost::scoped_ptr<idbdatafile::SMDataFile> df(
dynamic_cast<idbdatafile::SMDataFile *>(ffactory.open(fname, "w", 0, 0)));
if (!df)
{
int l_errno = errno;
cerr << "Failed to open/create " << &fname[prefixlen] << ": " <<
strerror_r(l_errno, (char *) data, 8192) << endl;
exit(1);
}
do
{
read_err = ::read(STDIN_FILENO, data, 8192);
if (read_err < 0)
read_err = ::read(STDIN_FILENO, data, 8192);
if (read_err < 0)
{
int l_errno = errno;
cerr << "Error reading stdin: " << strerror_r(l_errno, (char*)data, 8192) << endl;
exit(1);
}
count = 0;
while (count < read_err)
{
write_err = ioc->write(fname, &data[count], offset + count, read_err - count);
if (write_err < 0)
{
int l_errno = errno;
cerr << "Error reading stdin: " << strerror_r(l_errno, (char *) data, 8192) << endl;
exit(1);
}
count = 0;
while (count < read_err)
{
write_err = df->write(&data[count], read_err - count);
if (write_err < 0)
{
int l_errno = errno;
cerr << "Error writing to " << &fname[prefixlen] << ": " <<
strerror_r(l_errno, (char *) data, 8192) << endl;
exit(1);
}
count += write_err;
int l_errno = errno;
cerr << "Error writing to " << &fname[prefixlen] << ": " << strerror_r(l_errno, (char*)data, 8192)
<< endl;
exit(1);
}
count += write_err;
}
offset += read_err;
} while (read_err > 0);
}
catch (exception& e)
{
cerr << "smput putOffline FAIL: " << e.what() << endl;
}
}
int makePathPrefix(char *target, int targetlen)
void putOnline(const char* fname, int prefixlen)
{
// MCOL-3438 -> add bogus directories to the front of each param
Config *config = Config::get();
int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth"));
target[0] = '/';
target[1] = 0;
int bufpos = 1;
for (int i = 0; i < prefixDepth; i++)
uint8_t data[8192];
int read_err, write_err;
ssize_t count;
idbdatafile::SMFileFactory ffactory;
boost::scoped_ptr<idbdatafile::SMDataFile> df(
dynamic_cast<idbdatafile::SMDataFile*>(ffactory.open(fname, "w", 0, 0)));
if (!df)
{
int l_errno = errno;
cerr << "Failed to open/create " << &fname[prefixlen] << ": " << strerror_r(l_errno, (char*)data, 8192)
<< endl;
exit(1);
}
do
{
read_err = ::read(STDIN_FILENO, data, 8192);
if (read_err < 0)
{
if (bufpos + 3 >= targetlen)
{
cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth";
exit(1);
}
memcpy(&target[bufpos], "x/\0", 3);
bufpos += 2;
int l_errno = errno;
cerr << "Error reading stdin: " << strerror_r(l_errno, (char*)data, 8192) << endl;
exit(1);
}
return bufpos;
count = 0;
while (count < read_err)
{
write_err = df->write(&data[count], read_err - count);
if (write_err < 0)
{
int l_errno = errno;
cerr << "Error writing to " << &fname[prefixlen] << ": " << strerror_r(l_errno, (char*)data, 8192)
<< endl;
exit(1);
}
count += write_err;
}
} while (read_err > 0);
}
int main(int argc, char **argv)
int makePathPrefix(char* target, int targetlen)
{
if (argc != 2)
// MCOL-3438 -> add bogus directories to the front of each param
Config* config = Config::get();
int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth"));
target[0] = '/';
target[1] = 0;
int bufpos = 1;
for (int i = 0; i < prefixDepth; i++)
{
if (bufpos + 3 >= targetlen)
{
usage(argv[0]);
return 1;
cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth";
exit(1);
}
char prefix[8192];
int prefixlen = makePathPrefix(prefix, 8192);
if (SMOnline())
putOnline(strncat(prefix, argv[1], 8192 - prefixlen), prefixlen);
else
putOffline(strncat(prefix, argv[1], 8192 - prefixlen), prefixlen);
return 0;
memcpy(&target[bufpos], "x/\0", 3);
bufpos += 2;
}
return bufpos;
}
int main(int argc, char** argv)
{
if (argc != 2)
{
usage(argv[0]);
return 1;
}
char prefix[8192];
int prefixlen = makePathPrefix(prefix, 8192);
if (SMOnline())
putOnline(strncat(prefix, argv[1], 8192 - prefixlen), prefixlen);
else
putOffline(strncat(prefix, argv[1], 8192 - prefixlen), prefixlen);
return 0;
}

146
storage-manager/src/smrm.cpp
View File

@ -24,103 +24,103 @@
#include <sys/socket.h>
#include <sys/un.h>
using namespace std;
using namespace storagemanager;
void usage(const char *progname)
void usage(const char* progname)
{
cerr << progname << " is like 'rm -rf' for files managed by StorageManager, but with no options or globbing" << endl;
cerr << "Usage: " << progname << " file-or-dir1 file-or-dir2 .. file-or-dirN" << endl;
cerr << progname << " is like 'rm -rf' for files managed by StorageManager, but with no options or globbing"
<< endl;
cerr << "Usage: " << progname << " file-or-dir1 file-or-dir2 .. file-or-dirN" << endl;
}
bool SMOnline()
{
struct sockaddr_un addr;
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null...
int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0);
int err = ::connect(clientSocket, (const struct sockaddr *) &addr, sizeof(addr));
if (err >= 0)
{
::close(clientSocket);
return true;
}
return false;
struct sockaddr_un addr;
memset(&addr, 0, sizeof(addr));
addr.sun_family = AF_UNIX;
strcpy(&addr.sun_path[1], &socket_name[1]); // first char is null...
int clientSocket = ::socket(AF_UNIX, SOCK_STREAM, 0);
int err = ::connect(clientSocket, (const struct sockaddr*)&addr, sizeof(addr));
if (err >= 0)
{
::close(clientSocket);
return true;
}
return false;
}
#define min(x, y) (x < y ? x : y)
void rmOffline(int argCount, const char **args, const char *prefix, uint prefixlen)
void rmOffline(int argCount, const char** args, const char* prefix, uint prefixlen)
{
try
{
boost::scoped_ptr<IOCoordinator> ioc(IOCoordinator::get());
char buf[16384];
strncpy(buf, prefix, prefixlen);
for (int i = 1; i < argCount; i++)
{
memcpy(&buf[prefixlen], args[i], min(16383 - prefixlen, strlen(args[i])) + 1);
ioc->unlink(buf);
}
}
catch (exception &e)
{
cerr << "smrm rmOffline FAIL: " << e.what() << endl;
}
}
void rmOnline(int argCount, const char **args, const char *prefix, uint prefixlen)
{
idbdatafile::SMFileSystem fs;
try
{
boost::scoped_ptr<IOCoordinator> ioc(IOCoordinator::get());
char buf[16384];
strncpy(buf, prefix, prefixlen);
for (int i = 1; i < argCount; i++)
{
memcpy(&buf[prefixlen], args[i], min(16383 - prefixlen, strlen(args[i])) + 1);
fs.remove((char *) memcpy(&buf[prefixlen], args[i], min(16383 - prefixlen, strlen(args[i])) + 1));
memcpy(&buf[prefixlen], args[i], min(16383 - prefixlen, strlen(args[i])) + 1);
ioc->unlink(buf);
}
}
catch (exception& e)
{
cerr << "smrm rmOffline FAIL: " << e.what() << endl;
}
}
int makePathPrefix(char *target, int targetlen)
void rmOnline(int argCount, const char** args, const char* prefix, uint prefixlen)
{
// MCOL-3438 -> add bogus directories to the front of each param
Config *config = Config::get();
int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth"));
target[0] = '/';
target[1] = 0;
int bufpos = 1;
for (int i = 0; i < prefixDepth; i++)
{
if (bufpos + 3 >= targetlen)
{
cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth";
exit(1);
}
memcpy(&target[bufpos], "x/\0", 3);
bufpos += 2;
}
return bufpos;
idbdatafile::SMFileSystem fs;
char buf[16384];
strncpy(buf, prefix, prefixlen);
for (int i = 1; i < argCount; i++)
{
memcpy(&buf[prefixlen], args[i], min(16383 - prefixlen, strlen(args[i])) + 1);
fs.remove((char*)memcpy(&buf[prefixlen], args[i], min(16383 - prefixlen, strlen(args[i])) + 1));
}
}
int main(int argc, const char **argv)
int makePathPrefix(char* target, int targetlen)
{
if (argc < 2)
// MCOL-3438 -> add bogus directories to the front of each param
Config* config = Config::get();
int prefixDepth = stoi(config->getValue("ObjectStorage", "common_prefix_depth"));
target[0] = '/';
target[1] = 0;
int bufpos = 1;
for (int i = 0; i < prefixDepth; i++)
{
if (bufpos + 3 >= targetlen)
{
usage(argv[0]);
return 1;
cerr << "invalid prefix depth in ObjectStorage/common_prefix_depth";
exit(1);
}
char prefix[8192];
uint prefixlen = makePathPrefix(prefix, 8192);
if (SMOnline())
rmOnline(argc, argv, prefix, prefixlen);
else
rmOffline(argc, argv, prefix, prefixlen);
return 0;
memcpy(&target[bufpos], "x/\0", 3);
bufpos += 2;
}
return bufpos;
}
int main(int argc, const char** argv)
{
if (argc < 2)
{
usage(argv[0]);
return 1;
}
char prefix[8192];
uint prefixlen = makePathPrefix(prefix, 8192);
if (SMOnline())
rmOnline(argc, argv, prefix, prefixlen);
else
rmOffline(argc, argv, prefix, prefixlen);
return 0;
}

65
storage-manager/src/testS3Connection.cpp
View File

@ -24,45 +24,46 @@
using namespace storagemanager;
using namespace std;
void printUsage() {
cout << "MariaDB Columnstore Storage Manager Test Configuration Connectivity.\n" << endl;
cout << "Usage: testS3Connection \n" << endl;
cout << "Returns Success=0 Failure=1\n" << endl;
void printUsage()
{
cout << "MariaDB Columnstore Storage Manager Test Configuration Connectivity.\n" << endl;
cout << "Usage: testS3Connection \n" << endl;
cout << "Returns Success=0 Failure=1\n" << endl;
}
int s3TestConnection()
{
S3Storage* s3 = NULL;
int ret = 0;
try
{
S3Storage* s3 = new S3Storage(true);
cout << "S3 Storage Manager Configuration OK" << endl;
delete s3;
}
catch (exception &e)
{
cout << "S3 Storage Manager Configuration Error:" << endl;
cout << e.what() << endl;
if (s3)
delete s3;
ret = 1;
}
return ret;
S3Storage* s3 = NULL;
int ret = 0;
try
{
S3Storage* s3 = new S3Storage(true);
cout << "S3 Storage Manager Configuration OK" << endl;
delete s3;
}
catch (exception& e)
{
cout << "S3 Storage Manager Configuration Error:" << endl;
cout << e.what() << endl;
if (s3)
delete s3;
ret = 1;
}
return ret;
}
int main(int argc, char* argv[])
{
int option;
while ((option = getopt(argc, argv, "h")) != EOF )
int option;
while ((option = getopt(argc, argv, "h")) != EOF)
{
switch (option)
{
switch (option)
{
case 'h':
default:
printUsage();
return 0;
break;
}
case 'h':
default:
printUsage();
return 0;
break;
}
return s3TestConnection();
}
return s3TestConnection();
}

3131
storage-manager/src/unit_tests.cpp
View File

File diff suppressed because it is too large Load Diff