database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-12-09 10:41:14 +03:00
Code Activity

Checkpoint of Path changes MCOL-3365

Browse Source
This commit is contained in:
Ben Thompson
2019-07-24 10:06:47 -05:00
committed by benthompson15
parent 441c5eb62a
commit 8d68ff3847
5 changed files with 47 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
src/IOCoordinator.cpp
View File

@@ -158,12 +158,12 @@ ssize_t IOCoordinator::read(const char *_filename, uint8_t *data, off_t offset,
release read lock release read lock
put together the response in data put together the response in data
*/ */
bf::path p = ownership.get(_filename); bf::path filename = ownership.get(_filename);
const bf::path firstDir = *(p.begin()); const bf::path firstDir = *(filename.begin());
const char *filename = p.string().c_str(); //const char *filename = p.string().c_str();
ScopedReadLock fileLock(this, filename); ScopedReadLock fileLock(this, filename.string());
MetadataFile meta(filename, MetadataFile::no_create_t()); MetadataFile meta(filename, MetadataFile::no_create_t(),true);
if (!meta.exists()) if (!meta.exists())
{ {
@@ -283,18 +283,17 @@ out:
ssize_t IOCoordinator::write(const char *_filename, const uint8_t *data, off_t offset, size_t length) ssize_t IOCoordinator::write(const char *_filename, const uint8_t *data, off_t offset, size_t length)
{ {
bf::path p = ownership.get(_filename); bf::path filename = ownership.get(_filename);
const bf::path firstDir = *(p.begin()); const bf::path firstDir = *(filename.begin());
const char *filename = p.string().c_str();
bytesWritten += length; bytesWritten += length;
ScopedWriteLock lock(this, filename); ScopedWriteLock lock(this, filename.string());
int ret = _write(filename, data, offset, length, firstDir); int ret = _write(filename, data, offset, length, firstDir);
lock.unlock(); lock.unlock();
cache->doneWriting(firstDir); cache->doneWriting(firstDir);
return ret; return ret;
} }
ssize_t IOCoordinator::_write(const char *filename, const uint8_t *data, off_t offset, size_t length, ssize_t IOCoordinator::_write(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length,
const bf::path &firstDir) const bf::path &firstDir)
{ {
int err = 0; int err = 0;
@@ -306,7 +305,7 @@ ssize_t IOCoordinator::_write(const char *filename, const uint8_t *data, off_t o
vector<string> newObjectKeys; vector<string> newObjectKeys;
Synchronizer *synchronizer = Synchronizer::get(); // need to init sync here to break circular dependency... Synchronizer *synchronizer = Synchronizer::get(); // need to init sync here to break circular dependency...
MetadataFile metadata = MetadataFile(filename, MetadataFile::no_create_t()); MetadataFile metadata = MetadataFile(filename, MetadataFile::no_create_t(),true);
if (!metadata.exists()) if (!metadata.exists())
{ {
errno = ENOENT; errno = ENOENT;
@@ -421,9 +420,9 @@ ssize_t IOCoordinator::_write(const char *filename, const uint8_t *data, off_t o
ssize_t IOCoordinator::append(const char *_filename, const uint8_t *data, size_t length) ssize_t IOCoordinator::append(const char *_filename, const uint8_t *data, size_t length)
{ {
bf::path p = ownership.get(_filename); bf::path filename = ownership.get(_filename);
const bf::path firstDir = *(p.begin()); const bf::path firstDir = *(filename.begin());
const char *filename = p.string().c_str(); //const char *filename = p.string().c_str();
int err; int err;
ssize_t count = 0; ssize_t count = 0;
@@ -433,9 +432,9 @@ ssize_t IOCoordinator::append(const char *_filename, const uint8_t *data, size_t
vector<string> newObjectKeys; vector<string> newObjectKeys;
Synchronizer *synchronizer = Synchronizer::get(); // need to init sync here to break circular dependency... Synchronizer *synchronizer = Synchronizer::get(); // need to init sync here to break circular dependency...
ScopedWriteLock lock(this, filename); ScopedWriteLock lock(this, filename.string());
MetadataFile metadata = MetadataFile(filename, MetadataFile::no_create_t()); MetadataFile metadata = MetadataFile(filename, MetadataFile::no_create_t(),true);
if (!metadata.exists()) if (!metadata.exists())
{ {
errno = EBADF; errno = EBADF;
@@ -530,23 +529,23 @@ out:
// TODO: might need to support more open flags, ex: O_EXCL // TODO: might need to support more open flags, ex: O_EXCL
int IOCoordinator::open(const char *_filename, int openmode, struct stat *out) int IOCoordinator::open(const char *_filename, int openmode, struct stat *out)
{ {
bf::path p = ownership.get(_filename); bf::path filename = ownership.get(_filename);
const char *filename = p.string().c_str(); //const char *filename = p.string().c_str();
boost::scoped_ptr<ScopedFileLock> s; boost::scoped_ptr<ScopedFileLock> s;
if (openmode & O_CREAT || openmode | O_TRUNC) if (openmode & O_CREAT || openmode | O_TRUNC)
s.reset(new ScopedWriteLock(this, filename)); s.reset(new ScopedWriteLock(this, filename.string()));
else else
s.reset(new ScopedReadLock(this, filename)); s.reset(new ScopedReadLock(this, filename.string()));
MetadataFile meta(filename, MetadataFile::no_create_t()); MetadataFile meta(filename, MetadataFile::no_create_t(),true);
if ((openmode & O_CREAT) && !meta.exists()) { if ((openmode & O_CREAT) && !meta.exists()) {
++filesCreated; ++filesCreated;
replicator->updateMetadata(filename, meta); // this will end up creating filename replicator->updateMetadata(filename, meta); // this will end up creating filename
} }
if ((openmode & O_TRUNC) && meta.exists()) if ((openmode & O_TRUNC) && meta.exists())
_truncate(p, 0, s.get()); _truncate(filename, 0, s.get());
++filesOpened; ++filesOpened;
return meta.stat(out); return meta.stat(out);
@@ -582,14 +581,13 @@ int IOCoordinator::listDirectory(const char *dirname, vector<string> *listing)
int IOCoordinator::stat(const char *_path, struct stat *out) int IOCoordinator::stat(const char *_path, struct stat *out)
{ {
bf::path p = ownership.get(_path); bf::path filename = ownership.get(_path);
const char *path = p.string().c_str();
if (bf::is_directory(metaPath/p)) if (bf::is_directory(metaPath/filename))
return ::stat((metaPath/p).string().c_str(), out); return ::stat((metaPath/filename).string().c_str(), out);
ScopedReadLock s(this, path); ScopedReadLock s(this, filename.string());
MetadataFile meta(path, MetadataFile::no_create_t()); MetadataFile meta(filename, MetadataFile::no_create_t(),true);
return meta.stat(out); return meta.stat(out);
} }
@@ -617,12 +615,11 @@ int IOCoordinator::_truncate(const bf::path &bfpath, size_t newSize, ScopedFileL
tell synchronizer they were deleted tell synchronizer they were deleted
*/ */
const bf::path firstDir = *(bfpath.begin()); const bf::path firstDir = *(bfpath.begin());
const char *path = bfpath.string().c_str();
Synchronizer *synchronizer = Synchronizer::get(); // needs to init sync here to break circular dependency... Synchronizer *synchronizer = Synchronizer::get(); // needs to init sync here to break circular dependency...
int err; int err;
MetadataFile meta(path, MetadataFile::no_create_t()); MetadataFile meta(bfpath, MetadataFile::no_create_t(),true);
if (!meta.exists()) if (!meta.exists())
{ {
errno = ENOENT; errno = ENOENT;
@@ -637,7 +634,7 @@ int IOCoordinator::_truncate(const bf::path &bfpath, size_t newSize, ScopedFileL
if (filesize < newSize) if (filesize < newSize)
{ {
uint8_t zero = 0; uint8_t zero = 0;
err = _write(path, &zero, newSize - 1, 1, firstDir); err = _write(bfpath, &zero, newSize - 1, 1, firstDir);
lock->unlock(); lock->unlock();
cache->doneWriting(firstDir); cache->doneWriting(firstDir);
if (err < 0) if (err < 0)
@@ -658,7 +655,7 @@ int IOCoordinator::_truncate(const bf::path &bfpath, size_t newSize, ScopedFileL
for (uint i = 1; i < objects.size(); i++) for (uint i = 1; i < objects.size(); i++)
meta.removeEntry(objects[i].offset); meta.removeEntry(objects[i].offset);
err = replicator->updateMetadata(path, meta); err = replicator->updateMetadata(filename.string().c_str(), meta);
if (err) if (err)
return err; return err;
//lock.unlock(); <-- ifExistsThenDelete() needs the file lock held during the call //lock.unlock(); <-- ifExistsThenDelete() needs the file lock held during the call
@@ -703,7 +700,7 @@ void IOCoordinator::deleteMetaFile(const bf::path &file)
ScopedWriteLock lock(this, pita); ScopedWriteLock lock(this, pita);
//cout << "file is " << file.string() << " locked on " << pita << endl; //cout << "file is " << file.string() << " locked on " << pita << endl;
MetadataFile meta(file); MetadataFile meta(file,MetadataFile::no_create_t(),false);
replicator->remove(file); replicator->remove(file);
//lock.unlock(); <-- ifExistsThenDelete() needs the file lock held during the call //lock.unlock(); <-- ifExistsThenDelete() needs the file lock held during the call
@@ -858,7 +855,7 @@ int IOCoordinator::copyFile(const char *_filename1, const char *_filename2)
ScopedReadLock lock(this, filename1); ScopedReadLock lock(this, filename1);
ScopedWriteLock lock2(this, filename2); ScopedWriteLock lock2(this, filename2);
MetadataFile meta1(metaFile1); MetadataFile meta1(metaFile1);
MetadataFile meta2(metaFile2.string().c_str(), MetadataFile::no_create_t()); MetadataFile meta2(metaFile2, MetadataFile::no_create_t(),false);
vector<metadataObject> objects = meta1.metadataRead(0, meta1.getLength()); vector<metadataObject> objects = meta1.metadataRead(0, meta1.getLength());
bytesCopied += meta1.getLength(); bytesCopied += meta1.getLength();

2
src/IOCoordinator.h
View File

@@ -92,7 +92,7 @@ class IOCoordinator : public boost::noncopyable
void deleteMetaFile(const boost::filesystem::path &file); void deleteMetaFile(const boost::filesystem::path &file);
int _truncate(const boost::filesystem::path &path, size_t newsize, ScopedFileLock *lock); int _truncate(const boost::filesystem::path &path, size_t newsize, ScopedFileLock *lock);
ssize_t _write(const char *filename, const uint8_t *data, off_t offset, size_t length, ssize_t _write(const boost::filesystem::path &filename, const uint8_t *data, off_t offset, size_t length,
const boost::filesystem::path &firstDir); const boost::filesystem::path &firstDir);
int loadObjectAndJournal(const char *objFilename, const char *journalFilename, int loadObjectAndJournal(const char *objFilename, const char *journalFilename,

13
src/MetadataFile.cpp
View File

@@ -90,7 +90,7 @@ MetadataFile::MetadataFile()
} }
MetadataFile::MetadataFile(const char* filename) MetadataFile::MetadataFile(const boost::filesystem::path &filename)
{ {
mpConfig = MetadataConfig::get(); mpConfig = MetadataConfig::get();
mpLogger = SMLogging::get(); mpLogger = SMLogging::get();
@@ -124,12 +124,13 @@ MetadataFile::MetadataFile(const char* filename)
++metadataFilesAccessed; ++metadataFilesAccessed;
} }
MetadataFile::MetadataFile(const char* filename, no_create_t) MetadataFile::MetadataFile(const boost::filesystem::path &filename, no_create_t,bool appendExt)
{ {
mpConfig = MetadataConfig::get(); mpConfig = MetadataConfig::get();
mpLogger = SMLogging::get(); mpLogger = SMLogging::get();
mFilename = mpConfig->msMetadataPath + "/" + string(filename) + ".meta"; if(appendExt)
mFilename = mpConfig->msMetadataPath / string(filename) + ".meta";
if (boost::filesystem::exists(mFilename)) if (boost::filesystem::exists(mFilename))
{ {
_exists = true; _exists = true;
@@ -262,7 +263,7 @@ vector<metadataObject> MetadataFile::metadataRead(off_t offset, size_t length) c
return ret; return ret;
} }
metadataObject MetadataFile::addMetadataObject(const char *filename, size_t length) metadataObject MetadataFile::addMetadataObject(const boost::filesystem::path &filename, size_t length)
{ {
// this needs to handle if data write is beyond the end of the last object // this needs to handle if data write is beyond the end of the last object
// but not at start of new object // but not at start of new object
@@ -286,11 +287,11 @@ metadataObject MetadataFile::addMetadataObject(const char *filename, size_t leng
} }
int MetadataFile::writeMetadata(const char *filename) int MetadataFile::writeMetadata(const boost::filesystem::path &filename)
{ {
int error=0; int error=0;
string metadataFilename = mpConfig->msMetadataPath + "/" + string(filename) + ".meta"; string metadataFilename = mpConfig->msMetadataPath / filename + ".meta";
boost::filesystem::path pMetadataFilename = metadataFilename; boost::filesystem::path pMetadataFilename = metadataFilename;
boost::property_tree::ptree jsontree; boost::property_tree::ptree jsontree;
boost::property_tree::ptree objs; boost::property_tree::ptree objs;

12
src/MetadataFile.h
View File

@@ -32,13 +32,13 @@ class MetadataFile
public: public:
struct no_create_t {}; struct no_create_t {};
MetadataFile(); MetadataFile();
MetadataFile(const char* filename); MetadataFile(const boost::filesystem::path &filename);
MetadataFile(const char* filename, no_create_t); // this one won't create it if it doesn't exist MetadataFile(const boost::filesystem::path &path, no_create_t,bool appendExt); // this one won't create it if it doesn't exist
// this ctor is 'special'. It will take an absolute path, and it will assume it points to a metafile // this ctor is 'special'. It will take an absolute path, and it will assume it points to a metafile
// meaning, that it doesn't need the metadata prefix prepended, or the .meta extension appended. // meaning, that it doesn't need the metadata prefix prepended, or the .meta extension appended.
// aside from that, it will behave like the no_create ctor variant above // aside from that, it will behave like the no_create ctor variant above
MetadataFile(const boost::filesystem::path &path); //MetadataFile(const boost::filesystem::path &path);
~MetadataFile(); ~MetadataFile();
bool exists() const; bool exists() const;
@@ -48,12 +48,12 @@ class MetadataFile
// returns the objects needed to update // returns the objects needed to update
std::vector<metadataObject> metadataRead(off_t offset, size_t length) const; std::vector<metadataObject> metadataRead(off_t offset, size_t length) const;
// updates the metadatafile with new object // updates the metadatafile with new object
int writeMetadata(const char *filename); int writeMetadata(const boost::filesystem::path &filename);
// updates the name and length fields of an entry, given the offset // updates the name and length fields of an entry, given the offset
void updateEntry(off_t offset, const std::string &newName, size_t newLength); void updateEntry(off_t offset, const std::string &newName, size_t newLength);
void updateEntryLength(off_t offset, size_t newLength); void updateEntryLength(off_t offset, size_t newLength);
metadataObject addMetadataObject(const char *filename, size_t length); metadataObject addMetadataObject(const boost::filesystem::path &filename, size_t length);
bool getEntry(off_t offset, const metadataObject **out) const; bool getEntry(off_t offset, const metadataObject **out) const;
void removeEntry(off_t offset); void removeEntry(off_t offset);
void removeAllEntries(); void removeAllEntries();
@@ -92,7 +92,7 @@ class MetadataFile
SMLogging *mpLogger; SMLogging *mpLogger;
int mVersion; int mVersion;
int mRevision; int mRevision;
std::string mFilename; boost::filesystem::path mFilename;
std::set<metadataObject> mObjects; std::set<metadataObject> mObjects;
bool _exists; bool _exists;
}; };

4
src/Synchronizer.cpp
View File

@@ -423,7 +423,7 @@ void Synchronizer::synchronize(const string &sourceFile, list<string>::iterator
char buf[80]; char buf[80];
bool exists = false; bool exists = false;
int err; int err;
MetadataFile md(sourceFile.c_str(), MetadataFile::no_create_t()); MetadataFile md(sourceFile.c_str(), MetadataFile::no_create_t(),true);
if (!md.exists()) if (!md.exists())
{ {
@@ -481,7 +481,7 @@ void Synchronizer::synchronizeWithJournal(const string &sourceFile, list<string>
bf::path prefix = key.substr(0, pos); bf::path prefix = key.substr(0, pos);
string cloudKey = key.substr(pos + 1); string cloudKey = key.substr(pos + 1);
MetadataFile md(sourceFile.c_str(), MetadataFile::no_create_t()); MetadataFile md(sourceFile.c_str(), MetadataFile::no_create_t(),true);
if (!md.exists()) if (!md.exists())
{ {