database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-12-13 23:02:14 +03:00
Code Activity

Finished initial cut of the Cache class

Browse Source
This commit is contained in:
Patrick LeBlanc
2019-03-07 16:14:21 -06:00
parent df6675db01
commit 0626b30c88
9 changed files with 67 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
src/Cache.cpp
View File

@@ -104,20 +104,24 @@ void Cache::read(const vector<string> &keys)
// not in the cache, put it in the list to download
keysToFetch.push_back(&key);
}
// TODO: get the sizes of the objects to download and make space
// For now using an estimate
makeSpace(keys.size() * objectSize);
s.unlock();
// start downloading the keys to fetch
int dl_err;
vector<int> dl_errnos;
vector<size_t> sizes;
if (!keysToFetch.empty())
dl_err = downloader.download(keysToFetch, &dl_errnos);
dl_err = downloader.download(keysToFetch, &dl_errnos, &sizes);
size_t sum_sizes = 0;
for (size_t &size : sizes)
sum_sizes += size;
s.lock();
// do makespace() before downloading. Problem is, until the download is finished, this fcn can't tell which
// downloads it was responsible for. Need Downloader to make the call...?
makeSpace(sum_sizes);
currentCacheSize += sum_sizes;
// move all keys to the back of the LRU
for (i = 0; i < keys.size(); i++)
@@ -131,7 +135,8 @@ void Cache::read(const vector<string> &keys)
else if (dl_errnos[i] == 0) // successful download
{
lru.push_back(keys[i]);
m_lru.insert(M_LRU_element_t(&(lru.back()), lru.end()--));
LRU_t::iterator it = lru.end();
m_lru.insert(M_LRU_element_t(--it));
}
else
{
@@ -189,14 +194,31 @@ void Cache::exists(const vector<string> &keys, vector<bool> *out)
void Cache::newObject(const string &key, size_t size)
{
boost::unique_lock<boost::mutex> s(lru_mutex);
assert(m_lru.find(key) == m_lru.end());
makeSpace(size);
lru.push_back(key);
LRU_t::iterator back = lru.end();
m_lru.insert(--back);
currentCacheSize += size;
}
void Cache::deletedObject(const string &key, size_t size)
{
boost::unique_lock<boost::mutex> s(lru_mutex);
M_LRU_t::iterator mit = m_lru.find(key);
assert(mit != m_lru.end());
assert(doNotEvict.find(mit->lit) == doNotEvict.end());
lru.erase(mit->lit);
m_lru.erase(mit);
currentCacheSize -= size;
}
void Cache::setMaxCacheSize(size_t size)
{
if (size < maxCacheSize)
makeSpace(maxCacheSize - size);
maxCacheSize = size;
}
// call this holding lru_mutex
@@ -251,7 +273,7 @@ Cache::M_LRU_element_t::M_LRU_element_t(const string *k) : key(k)
Cache::M_LRU_element_t::M_LRU_element_t(const string &k) : key(&k)
{}
Cache::M_LRU_element_t::M_LRU_element_t(const string *k, const LRU_t::iterator &i) : key(k), lit(i)
Cache::M_LRU_element_t::M_LRU_element_t(const LRU_t::iterator &i) : key(&(*i)), lit(i)
{}
inline size_t Cache::KeyHasher::operator()(const M_LRU_element_t &l) const

2
src/Cache.h
View File

@@ -52,7 +52,7 @@ class Cache : public boost::noncopyable
{
M_LRU_element_t(const std::string &);
M_LRU_element_t(const std::string *);
M_LRU_element_t(const std::string *, const LRU_t::iterator &);
M_LRU_element_t(const LRU_t::iterator &);
const std::string *key;
LRU_t::iterator lit;
};

2
src/CloudStorage.h
View File

@@ -11,7 +11,7 @@ class CloudStorage
{
public:
/* These behave like syscalls. return code -1 means an error, and errno is set */
virtual int getObject(const std::string &sourceKey, const std::string &destFile) = 0;
virtual int getObject(const std::string &sourceKey, const std::string &destFile, size_t *size = NULL) = 0;
virtual int putObject(const std::string &sourceFile, const std::string &destKey) = 0;
virtual void deleteObject(const std::string &key) = 0;
virtual int copyObject(const std::string &sourceKey, const std::string &destKey) = 0;

22
src/Downloader.cpp
View File

@@ -36,9 +36,10 @@ inline boost::mutex & Downloader::getDownloadMutex()
return download_mutex;
}
int Downloader::download(const vector<const string *> &keys, vector<int> *errnos)
int Downloader::download(const vector<const string *> &keys, vector<int> *errnos, vector<size_t> *sizes)
{
volatile uint counter = keys.size();
//volatile uint counter = keys.size();
uint counter = keys.size();
boost::condition condvar;
boost::mutex m;
DownloadListener listener(&counter, &condvar, &m);
@@ -71,19 +72,27 @@ int Downloader::download(const vector<const string *> &keys, vector<int> *errnos
(*iterators[i])->listeners.push_back(&listener);
}
}
s.unlock();
// wait for the downloads to finish
boost::unique_lock<boost::mutex> dl_lock(m);
s.unlock();
while (counter > 0)
condvar.wait(dl_lock);
dl_lock.unlock();
// remove the entries inserted by this call
sizes->resize(keys.size());
s.lock();
for (i = 0; i < keys.size(); i++)
{
if (inserted[i])
{
(*sizes)[i] = (*iterators[i])->size;
downloads.erase(iterators[i]);
}
else
(*sizes)[i] = 0;
}
s.unlock();
// check for errors & propagate
@@ -113,14 +122,14 @@ inline const string & Downloader::getDownloadPath() const
}
/* The helper fcns */
Downloader::Download::Download(const string *source, Downloader *dl) : key(source), dler(dl), dl_errno(0)
Downloader::Download::Download(const string *source, Downloader *dl) : key(source), dler(dl), dl_errno(0), size(0)
{
}
void Downloader::Download::operator()()
{
CloudStorage *storage = CloudStorage::get();
int err = storage->getObject(*key, dler->getDownloadPath() + "/" + *key);
int err = storage->getObject(*key, dler->getDownloadPath() + "/" + *key, &size);
if (err != 0)
dl_errno = errno;
@@ -129,7 +138,8 @@ void Downloader::Download::operator()()
listener->downloadFinished();
}
Downloader::DownloadListener::DownloadListener(volatile uint *counter, boost::condition *condvar, boost::mutex *m) : count(counter), cond(condvar), mutex(m)
//Downloader::DownloadListener::DownloadListener(volatile uint *counter, boost::condition *condvar, boost::mutex *m) : count(counter), cond(condvar), mutex(m)
Downloader::DownloadListener::DownloadListener(uint *counter, boost::condition *condvar, boost::mutex *m) : count(counter), cond(condvar), mutex(m)
{
}

9
src/Downloader.h
View File

@@ -24,7 +24,7 @@ class Downloader
// returns 0 on success. If != 0, errnos will contains the errno associated with the failure
// caller owns the memory for the strings.
int download(const std::vector<const std::string *> &keys, std::vector<int> *errnos);
int download(const std::vector<const std::string *> &keys, std::vector<int> *errnos, std::vector<size_t> *sizes);
void setDownloadPath(const std::string &path);
const std::string & getDownloadPath() const;
@@ -35,10 +35,12 @@ class Downloader
class DownloadListener
{
public:
DownloadListener(volatile uint *counter, boost::condition *condvar, boost::mutex *m);
DownloadListener(uint *counter, boost::condition *condvar, boost::mutex *m);
//DownloadListener(volatile uint *counter, boost::condition *condvar, boost::mutex *m);
void downloadFinished();
private:
volatile uint *count;
uint *count;
//volatile uint *count;
boost::condition *cond;
boost::mutex *mutex;
};
@@ -50,6 +52,7 @@ class Downloader
Downloader *dler;
const std::string *key;
int dl_errno; // to propagate errors from the download job to the caller
size_t size;
std::vector<DownloadListener *> listeners;
};

11
src/LocalStorage.cpp
View File

@@ -43,7 +43,7 @@ int LocalStorage::copy(const path &source, const path &dest)
{
boost::system::error_code err;
copy_file(source, dest, copy_option::fail_if_exists, err);
if (!err)
if (err)
{
errno = err.value();
return -1;
@@ -58,9 +58,14 @@ path operator+(const path &p1, const path &p2)
return ret;
}
int LocalStorage::getObject(const string &source, const string &dest)
int LocalStorage::getObject(const string &source, const string &dest, size_t *size)
{
return copy(prefix + source, dest);
int ret = copy(prefix + source, dest);
if (ret)
return ret;
if (size)
*size = boost::filesystem::file_size(dest);
return ret;
}
int LocalStorage::putObject(const string &source, const string &dest)

2
src/LocalStorage.h
View File

@@ -14,7 +14,7 @@ class LocalStorage : public CloudStorage
LocalStorage();
virtual ~LocalStorage();
int getObject(const std::string &sourceKey, const std::string &destFile);
int getObject(const std::string &sourceKey, const std::string &destFile, size_t *size = NULL);
int putObject(const std::string &sourceFile, const std::string &destKey);
void deleteObject(const std::string &key);
int copyObject(const std::string &sourceKey, const std::string &destKey);

2
src/S3Storage.cpp
View File

@@ -14,7 +14,7 @@ S3Storage::~S3Storage()
{
}
int S3Storage::getObject(const string &sourceKey, const string &destFile)
int S3Storage::getObject(const string &sourceKey, const string &destFile, size_t *size)
{
return 0;
}

2
src/S3Storage.h
View File

@@ -14,7 +14,7 @@ class S3Storage : public CloudStorage
S3Storage();
virtual ~S3Storage();
int getObject(const std::string &sourceKey, const std::string &destFile);
int getObject(const std::string &sourceKey, const std::string &destFile, size_t *size = NULL);
int putObject(const std::string &sourceFile, const std::string &destKey);
void deleteObject(const std::string &key);
int copyObject(const std::string &sourceKey, const std::string &destKey);