You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-08-07 03:22:57 +03:00
clang format apply
This commit is contained in:
@@ -15,7 +15,6 @@
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
|
||||
MA 02110-1301, USA. */
|
||||
|
||||
|
||||
#include "Cache.h"
|
||||
#include "Config.h"
|
||||
#include "Downloader.h"
|
||||
@@ -33,330 +32,328 @@ namespace bf = boost::filesystem;
|
||||
|
||||
namespace
|
||||
{
|
||||
boost::mutex m;
|
||||
storagemanager::Cache *inst = NULL;
|
||||
}
|
||||
boost::mutex m;
|
||||
storagemanager::Cache* inst = NULL;
|
||||
} // namespace
|
||||
|
||||
namespace storagemanager
|
||||
{
|
||||
|
||||
Cache * Cache::get()
|
||||
Cache* Cache::get()
|
||||
{
|
||||
if (inst)
|
||||
return inst;
|
||||
boost::unique_lock<boost::mutex> s(m);
|
||||
if (inst)
|
||||
return inst;
|
||||
inst = new Cache();
|
||||
if (inst)
|
||||
return inst;
|
||||
boost::unique_lock<boost::mutex> s(m);
|
||||
if (inst)
|
||||
return inst;
|
||||
inst = new Cache();
|
||||
return inst;
|
||||
}
|
||||
|
||||
Cache::Cache()
|
||||
{
|
||||
Config *conf = Config::get();
|
||||
logger = SMLogging::get();
|
||||
|
||||
configListener();
|
||||
conf->addConfigListener(this);
|
||||
//cout << "Cache got cache size " << maxCacheSize << endl;
|
||||
|
||||
string stmp = conf->getValue("ObjectStorage", "object_size");
|
||||
if (stmp.empty())
|
||||
{
|
||||
logger->log(LOG_CRIT, "ObjectStorage/object_size is not set");
|
||||
throw runtime_error("Please set ObjectStorage/object_size in the storagemanager.cnf file");
|
||||
}
|
||||
try
|
||||
{
|
||||
objectSize = stoul(stmp);
|
||||
}
|
||||
catch (invalid_argument &)
|
||||
{
|
||||
logger->log(LOG_CRIT, "ObjectStorage/object_size is not a number");
|
||||
throw runtime_error("Please set ObjectStorage/object_size to a number");
|
||||
}
|
||||
|
||||
cachePrefix = conf->getValue("Cache", "path");
|
||||
if (cachePrefix.empty())
|
||||
{
|
||||
logger->log(LOG_CRIT, "Cache/path is not set");
|
||||
throw runtime_error("Please set Cache/path in the storagemanager.cnf file");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
bf::create_directories(cachePrefix);
|
||||
}
|
||||
catch (exception &e)
|
||||
{
|
||||
logger->log(LOG_CRIT, "Failed to create %s, got: %s", cachePrefix.string().c_str(), e.what());
|
||||
throw e;
|
||||
}
|
||||
//cout << "Cache got cachePrefix " << cachePrefix << endl;
|
||||
downloader.reset(new Downloader());
|
||||
|
||||
stmp = conf->getValue("ObjectStorage", "journal_path");
|
||||
if (stmp.empty())
|
||||
{
|
||||
logger->log(LOG_CRIT, "ObjectStorage/journal_path is not set");
|
||||
throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file");
|
||||
}
|
||||
journalPrefix = stmp;
|
||||
Config* conf = Config::get();
|
||||
logger = SMLogging::get();
|
||||
|
||||
configListener();
|
||||
conf->addConfigListener(this);
|
||||
// cout << "Cache got cache size " << maxCacheSize << endl;
|
||||
|
||||
string stmp = conf->getValue("ObjectStorage", "object_size");
|
||||
if (stmp.empty())
|
||||
{
|
||||
logger->log(LOG_CRIT, "ObjectStorage/object_size is not set");
|
||||
throw runtime_error("Please set ObjectStorage/object_size in the storagemanager.cnf file");
|
||||
}
|
||||
try
|
||||
{
|
||||
objectSize = stoul(stmp);
|
||||
}
|
||||
catch (invalid_argument&)
|
||||
{
|
||||
logger->log(LOG_CRIT, "ObjectStorage/object_size is not a number");
|
||||
throw runtime_error("Please set ObjectStorage/object_size to a number");
|
||||
}
|
||||
|
||||
cachePrefix = conf->getValue("Cache", "path");
|
||||
if (cachePrefix.empty())
|
||||
{
|
||||
logger->log(LOG_CRIT, "Cache/path is not set");
|
||||
throw runtime_error("Please set Cache/path in the storagemanager.cnf file");
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
bf::create_directories(cachePrefix);
|
||||
}
|
||||
catch (exception& e)
|
||||
{
|
||||
logger->log(LOG_CRIT, "Failed to create %s, got: %s", cachePrefix.string().c_str(), e.what());
|
||||
throw e;
|
||||
}
|
||||
// cout << "Cache got cachePrefix " << cachePrefix << endl;
|
||||
downloader.reset(new Downloader());
|
||||
|
||||
stmp = conf->getValue("ObjectStorage", "journal_path");
|
||||
if (stmp.empty())
|
||||
{
|
||||
logger->log(LOG_CRIT, "ObjectStorage/journal_path is not set");
|
||||
throw runtime_error("Please set ObjectStorage/journal_path in the storagemanager.cnf file");
|
||||
}
|
||||
journalPrefix = stmp;
|
||||
bf::create_directories(journalPrefix);
|
||||
try
|
||||
{
|
||||
bf::create_directories(journalPrefix);
|
||||
try
|
||||
{
|
||||
bf::create_directories(journalPrefix);
|
||||
}
|
||||
catch (exception &e)
|
||||
{
|
||||
logger->log(LOG_CRIT, "Failed to create %s, got: %s", journalPrefix.string().c_str(), e.what());
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
catch (exception& e)
|
||||
{
|
||||
logger->log(LOG_CRIT, "Failed to create %s, got: %s", journalPrefix.string().c_str(), e.what());
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
Cache::~Cache()
|
||||
{
|
||||
Config::get()->removeConfigListener(this);
|
||||
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
|
||||
delete it->second;
|
||||
Config::get()->removeConfigListener(this);
|
||||
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
|
||||
delete it->second;
|
||||
}
|
||||
|
||||
// be careful using this! SM should be idle. No ongoing reads or writes.
|
||||
void Cache::validateCacheSize()
|
||||
{
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
|
||||
it->second->validateCacheSize();
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
|
||||
it->second->validateCacheSize();
|
||||
}
|
||||
|
||||
// new simplified version
|
||||
void Cache::read(const bf::path &prefix, const vector<string> &keys)
|
||||
void Cache::read(const bf::path& prefix, const vector<string>& keys)
|
||||
{
|
||||
getPCache(prefix).read(keys);
|
||||
getPCache(prefix).read(keys);
|
||||
}
|
||||
|
||||
void Cache::doneReading(const bf::path &prefix, const vector<string> &keys)
|
||||
void Cache::doneReading(const bf::path& prefix, const vector<string>& keys)
|
||||
{
|
||||
getPCache(prefix).doneReading(keys);
|
||||
getPCache(prefix).doneReading(keys);
|
||||
}
|
||||
|
||||
void Cache::doneWriting(const bf::path &prefix)
|
||||
void Cache::doneWriting(const bf::path& prefix)
|
||||
{
|
||||
getPCache(prefix).makeSpace(0);
|
||||
getPCache(prefix).makeSpace(0);
|
||||
}
|
||||
|
||||
const bf::path & Cache::getCachePath() const
|
||||
const bf::path& Cache::getCachePath() const
|
||||
{
|
||||
return cachePrefix;
|
||||
return cachePrefix;
|
||||
}
|
||||
|
||||
const bf::path & Cache::getJournalPath() const
|
||||
const bf::path& Cache::getJournalPath() const
|
||||
{
|
||||
return journalPrefix;
|
||||
return journalPrefix;
|
||||
}
|
||||
|
||||
const bf::path Cache::getCachePath(const bf::path &prefix) const
|
||||
const bf::path Cache::getCachePath(const bf::path& prefix) const
|
||||
{
|
||||
return cachePrefix/prefix;
|
||||
return cachePrefix / prefix;
|
||||
}
|
||||
|
||||
const bf::path Cache::getJournalPath(const bf::path &prefix) const
|
||||
const bf::path Cache::getJournalPath(const bf::path& prefix) const
|
||||
{
|
||||
return journalPrefix/prefix;
|
||||
}
|
||||
|
||||
void Cache::exists(const bf::path &prefix, const vector<string> &keys, vector<bool> *out)
|
||||
{
|
||||
getPCache(prefix).exists(keys, out);
|
||||
return journalPrefix / prefix;
|
||||
}
|
||||
|
||||
bool Cache::exists(const bf::path &prefix, const string &key)
|
||||
void Cache::exists(const bf::path& prefix, const vector<string>& keys, vector<bool>* out)
|
||||
{
|
||||
return getPCache(prefix).exists(key);
|
||||
getPCache(prefix).exists(keys, out);
|
||||
}
|
||||
|
||||
void Cache::newObject(const bf::path &prefix, const string &key, size_t size)
|
||||
bool Cache::exists(const bf::path& prefix, const string& key)
|
||||
{
|
||||
getPCache(prefix).newObject(key, size);
|
||||
return getPCache(prefix).exists(key);
|
||||
}
|
||||
|
||||
void Cache::newJournalEntry(const bf::path &prefix, size_t size)
|
||||
void Cache::newObject(const bf::path& prefix, const string& key, size_t size)
|
||||
{
|
||||
getPCache(prefix).newJournalEntry(size);
|
||||
getPCache(prefix).newObject(key, size);
|
||||
}
|
||||
|
||||
void Cache::deletedJournal(const bf::path &prefix, size_t size)
|
||||
void Cache::newJournalEntry(const bf::path& prefix, size_t size)
|
||||
{
|
||||
getPCache(prefix).deletedJournal(size);
|
||||
getPCache(prefix).newJournalEntry(size);
|
||||
}
|
||||
|
||||
void Cache::deletedObject(const bf::path &prefix, const string &key, size_t size)
|
||||
void Cache::deletedJournal(const bf::path& prefix, size_t size)
|
||||
{
|
||||
getPCache(prefix).deletedObject(key, size);
|
||||
getPCache(prefix).deletedJournal(size);
|
||||
}
|
||||
|
||||
void Cache::deletedObject(const bf::path& prefix, const string& key, size_t size)
|
||||
{
|
||||
getPCache(prefix).deletedObject(key, size);
|
||||
}
|
||||
|
||||
void Cache::setMaxCacheSize(size_t size)
|
||||
{
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
maxCacheSize = size;
|
||||
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
|
||||
it->second->setMaxCacheSize(size);
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
maxCacheSize = size;
|
||||
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
|
||||
it->second->setMaxCacheSize(size);
|
||||
}
|
||||
|
||||
void Cache::makeSpace(const bf::path &prefix, size_t size)
|
||||
void Cache::makeSpace(const bf::path& prefix, size_t size)
|
||||
{
|
||||
getPCache(prefix).makeSpace(size);
|
||||
getPCache(prefix).makeSpace(size);
|
||||
}
|
||||
|
||||
size_t Cache::getMaxCacheSize() const
|
||||
{
|
||||
return maxCacheSize;
|
||||
return maxCacheSize;
|
||||
}
|
||||
|
||||
void Cache::rename(const bf::path &prefix, const string &oldKey, const string &newKey, ssize_t sizediff)
|
||||
void Cache::rename(const bf::path& prefix, const string& oldKey, const string& newKey, ssize_t sizediff)
|
||||
{
|
||||
getPCache(prefix).rename(oldKey, newKey, sizediff);
|
||||
getPCache(prefix).rename(oldKey, newKey, sizediff);
|
||||
}
|
||||
|
||||
int Cache::ifExistsThenDelete(const bf::path &prefix, const string &key)
|
||||
int Cache::ifExistsThenDelete(const bf::path& prefix, const string& key)
|
||||
{
|
||||
return getPCache(prefix).ifExistsThenDelete(key);
|
||||
return getPCache(prefix).ifExistsThenDelete(key);
|
||||
}
|
||||
|
||||
void Cache::printKPIs() const
|
||||
{
|
||||
downloader->printKPIs();
|
||||
downloader->printKPIs();
|
||||
}
|
||||
size_t Cache::getCurrentCacheSize()
|
||||
{
|
||||
size_t totalSize = 0;
|
||||
size_t totalSize = 0;
|
||||
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
|
||||
totalSize += it->second->getCurrentCacheSize();
|
||||
return totalSize;
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
|
||||
totalSize += it->second->getCurrentCacheSize();
|
||||
return totalSize;
|
||||
}
|
||||
|
||||
size_t Cache::getCurrentCacheElementCount()
|
||||
{
|
||||
size_t totalCount = 0;
|
||||
size_t totalCount = 0;
|
||||
|
||||
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
|
||||
totalCount += it->second->getCurrentCacheElementCount();
|
||||
return totalCount;
|
||||
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
|
||||
totalCount += it->second->getCurrentCacheElementCount();
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
size_t Cache::getCurrentCacheSize(const bf::path &prefix)
|
||||
size_t Cache::getCurrentCacheSize(const bf::path& prefix)
|
||||
{
|
||||
return getPCache(prefix).getCurrentCacheSize();
|
||||
return getPCache(prefix).getCurrentCacheSize();
|
||||
}
|
||||
|
||||
size_t Cache::getCurrentCacheElementCount(const bf::path &prefix)
|
||||
size_t Cache::getCurrentCacheElementCount(const bf::path& prefix)
|
||||
{
|
||||
return getPCache(prefix).getCurrentCacheElementCount();
|
||||
return getPCache(prefix).getCurrentCacheElementCount();
|
||||
}
|
||||
|
||||
void Cache::reset()
|
||||
{
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
|
||||
it->second->reset();
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
for (auto it = prefixCaches.begin(); it != prefixCaches.end(); ++it)
|
||||
it->second->reset();
|
||||
}
|
||||
|
||||
void Cache::newPrefix(const bf::path &prefix)
|
||||
void Cache::newPrefix(const bf::path& prefix)
|
||||
{
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
//cerr << "Cache: making new prefix " << prefix.string() << endl;
|
||||
assert(prefixCaches.find(prefix) == prefixCaches.end());
|
||||
prefixCaches[prefix] = NULL;
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
// cerr << "Cache: making new prefix " << prefix.string() << endl;
|
||||
assert(prefixCaches.find(prefix) == prefixCaches.end());
|
||||
prefixCaches[prefix] = NULL;
|
||||
s.unlock();
|
||||
PrefixCache* pCache = new PrefixCache(prefix);
|
||||
s.lock();
|
||||
prefixCaches[prefix] = pCache;
|
||||
}
|
||||
|
||||
void Cache::dropPrefix(const bf::path& prefix)
|
||||
{
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
auto* pCache = prefixCaches[prefix];
|
||||
prefixCaches.erase(prefix);
|
||||
s.unlock();
|
||||
delete pCache;
|
||||
}
|
||||
|
||||
inline PrefixCache& Cache::getPCache(const bf::path& prefix)
|
||||
{
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
// cerr << "Getting pcache for " << prefix.string() << endl;
|
||||
PrefixCache* ret;
|
||||
auto it = prefixCaches.find(prefix);
|
||||
assert(it != prefixCaches.end());
|
||||
|
||||
ret = it->second;
|
||||
while (ret == NULL) // wait for the new PC to init
|
||||
{
|
||||
s.unlock();
|
||||
PrefixCache *pCache = new PrefixCache(prefix);
|
||||
sleep(1);
|
||||
s.lock();
|
||||
prefixCaches[prefix] = pCache;
|
||||
ret = prefixCaches[prefix];
|
||||
}
|
||||
|
||||
return *ret;
|
||||
}
|
||||
|
||||
void Cache::dropPrefix(const bf::path &prefix)
|
||||
Downloader* Cache::getDownloader() const
|
||||
{
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
auto *pCache = prefixCaches[prefix];
|
||||
prefixCaches.erase(prefix);
|
||||
s.unlock();
|
||||
delete pCache;
|
||||
}
|
||||
|
||||
inline PrefixCache & Cache::getPCache(const bf::path &prefix)
|
||||
{
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
|
||||
//cerr << "Getting pcache for " << prefix.string() << endl;
|
||||
PrefixCache *ret;
|
||||
auto it = prefixCaches.find(prefix);
|
||||
assert(it != prefixCaches.end());
|
||||
|
||||
ret = it->second;
|
||||
while (ret == NULL) // wait for the new PC to init
|
||||
{
|
||||
s.unlock();
|
||||
sleep(1);
|
||||
s.lock();
|
||||
ret = prefixCaches[prefix];
|
||||
}
|
||||
|
||||
return *ret;
|
||||
}
|
||||
|
||||
Downloader * Cache::getDownloader() const
|
||||
{
|
||||
return downloader.get();
|
||||
return downloader.get();
|
||||
}
|
||||
|
||||
void Cache::shutdown()
|
||||
{
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
downloader.reset();
|
||||
boost::unique_lock<boost::mutex> s(lru_mutex);
|
||||
downloader.reset();
|
||||
}
|
||||
|
||||
void Cache::configListener()
|
||||
{
|
||||
Config *conf = Config::get();
|
||||
SMLogging* logger = SMLogging::get();
|
||||
Config* conf = Config::get();
|
||||
SMLogging* logger = SMLogging::get();
|
||||
|
||||
if (maxCacheSize == 0)
|
||||
maxCacheSize = 2147483648;
|
||||
string stmp = conf->getValue("Cache", "cache_size");
|
||||
if (stmp.empty())
|
||||
if (maxCacheSize == 0)
|
||||
maxCacheSize = 2147483648;
|
||||
string stmp = conf->getValue("Cache", "cache_size");
|
||||
if (stmp.empty())
|
||||
{
|
||||
logger->log(LOG_CRIT, "Cache/cache_size is not set. Using current value = %zi", maxCacheSize);
|
||||
}
|
||||
try
|
||||
{
|
||||
size_t newMaxCacheSize = stoull(stmp);
|
||||
if (newMaxCacheSize != maxCacheSize)
|
||||
{
|
||||
logger->log(LOG_CRIT, "Cache/cache_size is not set. Using current value = %zi",maxCacheSize);
|
||||
}
|
||||
try
|
||||
{
|
||||
size_t newMaxCacheSize = stoull(stmp);
|
||||
if (newMaxCacheSize != maxCacheSize)
|
||||
{
|
||||
if (newMaxCacheSize >= MIN_CACHE_SIZE)
|
||||
{
|
||||
setMaxCacheSize(newMaxCacheSize);
|
||||
logger->log(LOG_INFO, "Cache/cache_size = %zi",maxCacheSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
logger->log(LOG_CRIT, "Cache/cache_size is below %u. Check value and suffix are correct in configuration file. Using current value = %zi",MIN_CACHE_SIZE,maxCacheSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (invalid_argument &)
|
||||
{
|
||||
logger->log(LOG_CRIT, "Cache/cache_size is not a number. Using current value = %zi",maxCacheSize);
|
||||
if (newMaxCacheSize >= MIN_CACHE_SIZE)
|
||||
{
|
||||
setMaxCacheSize(newMaxCacheSize);
|
||||
logger->log(LOG_INFO, "Cache/cache_size = %zi", maxCacheSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
logger->log(LOG_CRIT,
|
||||
"Cache/cache_size is below %u. Check value and suffix are correct in configuration file. "
|
||||
"Using current value = %zi",
|
||||
MIN_CACHE_SIZE, maxCacheSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (invalid_argument&)
|
||||
{
|
||||
logger->log(LOG_CRIT, "Cache/cache_size is not a number. Using current value = %zi", maxCacheSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
} // namespace storagemanager
|
||||
|
Reference in New Issue
Block a user