diff --git a/oamapps/postConfigure/installer.cpp b/oamapps/postConfigure/installer.cpp index 50d9fb9fc..a0395bb57 100644 --- a/oamapps/postConfigure/installer.cpp +++ b/oamapps/postConfigure/installer.cpp @@ -989,7 +989,7 @@ int main(int argc, char* argv[]) system(cmd.c_str()); if (oam.checkLogStatus(logFile, "System Catalog created") ) - cout << endl << "System Catalog Successfull Created" << endl; + cout << endl << "System Catalog Successfully Created" << endl; else { if ( oam.checkLogStatus(logFile, "System catalog appears to exist") ) diff --git a/oamapps/postConfigure/postConfigure.cpp b/oamapps/postConfigure/postConfigure.cpp index b26c60b50..888db94f3 100644 --- a/oamapps/postConfigure/postConfigure.cpp +++ b/oamapps/postConfigure/postConfigure.cpp @@ -808,6 +808,7 @@ int main(int argc, char* argv[]) try { sysConfig->setConfig(InstallSection, "MySQLPort", mysqlPort); + sysConfig->setConfig("CrossEngineSupport", "Port", mysqlPort); } catch (...) {} diff --git a/oamapps/postConfigure/quick_installer_single_server.sh b/oamapps/postConfigure/quick_installer_single_server.sh index 1b747adc3..51db8a1ba 100755 --- a/oamapps/postConfigure/quick_installer_single_server.sh +++ b/oamapps/postConfigure/quick_installer_single_server.sh @@ -6,13 +6,13 @@ for arg in "$@"; do if [ `expr -- "$arg" : '--help'` -eq 6 ]; then - echo "Usage ./quick_installer_multi_server.sh" + echo "Usage ./quick_installer_single_server.sh" echo "" echo "Quick Installer for a Single Server MariaDB ColumnStore Install" echo "" exit 1 else - echo "quick_installer_multi_server.sh: ignoring unknown argument: $arg" 1>&2 + echo "quick_installer_single_server.sh: ignoring unknown argument: $arg" 1>&2 fi done diff --git a/versioning/BRM/blockresolutionmanager.cpp b/versioning/BRM/blockresolutionmanager.cpp index b957ca622..7127f3f43 100644 --- a/versioning/BRM/blockresolutionmanager.cpp +++ b/versioning/BRM/blockresolutionmanager.cpp @@ -109,20 +109,9 @@ int BlockResolutionManager::saveState(string filename) throw() // truncate teh file if already exists since no truncate in HDFS. const char* filename = journalFilename.c_str(); - if (true || IDBPolicy::useHdfs()) - { - IDBDataFile* journal = IDBDataFile::open( - IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "wb", 0); - delete journal; - } - else - { - ofstream journal; - uint32_t utmp = ::umask(0); - journal.open(filename, ios_base::out | ios_base::trunc | ios_base::binary); - journal.close(); - ::umask(utmp); - } + IDBDataFile* journal = IDBDataFile::open( + IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "wb", 0); + delete journal; vbbm.save(vbbmFilename); vss.save(vssFilename); diff --git a/versioning/BRM/extentmap.cpp b/versioning/BRM/extentmap.cpp index b7733097d..c17d49588 100644 --- a/versioning/BRM/extentmap.cpp +++ b/versioning/BRM/extentmap.cpp @@ -1133,84 +1133,6 @@ void ExtentMap::reserveLBIDRange(LBID_t start, uint8_t size) ... (* numFL) */ -void ExtentMap::loadVersion4(ifstream& in) -{ - int emNumElements, flNumElements; - - in.read((char*) &emNumElements, sizeof(int)); - in.read((char*) &flNumElements, sizeof(int)); - idbassert(emNumElements > 0); - - void *fExtentMapPtr = static_cast(fExtentMap); - memset(fExtentMapPtr, 0, fEMShminfo->allocdSize); - fEMShminfo->currentSize = 0; - - // init the free list - memset(fFreeList, 0, fFLShminfo->allocdSize); - fFreeList[0].size = (1 << 26); // 2^36 LBIDs - fFLShminfo->currentSize = sizeof(InlineLBIDRange); - - // @Bug 3498 - // Calculate how big an extent map we're going to need and allocate it in one call - if ((fEMShminfo->allocdSize / sizeof(EMEntry)) < (unsigned)emNumElements) - { - size_t nrows = emNumElements; - - //Round up to the nearest EM_INCREMENT_ROWS - if ((nrows % EM_INCREMENT_ROWS) != 0) - { - nrows /= EM_INCREMENT_ROWS; - nrows++; - nrows *= EM_INCREMENT_ROWS; - } - - growEMShmseg(nrows); - } - - for (int i = 0; i < emNumElements; i++) - { - in.read((char*) &fExtentMap[i], sizeof(EMEntry)); - reserveLBIDRange(fExtentMap[i].range.start, fExtentMap[i].range.size); - - //@bug 1911 - verify status value is valid - if (fExtentMap[i].status < EXTENTSTATUSMIN || - fExtentMap[i].status > EXTENTSTATUSMAX) - fExtentMap[i].status = EXTENTAVAILABLE; - } - - fEMShminfo->currentSize = emNumElements * sizeof(EMEntry); - -#ifdef DUMP_EXTENT_MAP - EMEntry* emSrc = fExtentMap; - cout << "lbid\tsz\toid\tfbo\thwm\tpart#\tseg#\tDBRoot\twid\tst\thi\tlo\tsq\tv" << endl; - - for (int i = 0; i < emNumElements; i++) - { - cout << - emSrc[i].start - << '\t' << emSrc[i].size - << '\t' << emSrc[i].fileID - << '\t' << emSrc[i].blockOffset - << '\t' << emSrc[i].HWM - << '\t' << emSrc[i].partitionNum - << '\t' << emSrc[i].segmentNum - << '\t' << emSrc[i].dbRoot - << '\t' << emSrc[i].status - << '\t' << emSrc[i].partition.cprange.hi_val - << '\t' << emSrc[i].partition.cprange.lo_val - << '\t' << emSrc[i].partition.cprange.sequenceNum - << '\t' << (int)(emSrc[i].partition.cprange.isValid) - << endl; - } - - cout << "Free list entries:" << endl; - cout << "start\tsize" << endl; - - for (int i = 0; i < flNumElements; i++) - cout << fFreeList[i].start << '\t' << fFreeList[i].size << endl; - -#endif -} void ExtentMap::loadVersion4(IDBDataFile* in) { @@ -1264,7 +1186,6 @@ void ExtentMap::loadVersion4(IDBDataFile* in) log_errno("ExtentMap::loadVersion4(): read "); throw runtime_error("ExtentMap::loadVersion4(): read failed. Check the error log."); } - progress += (uint) err; } @@ -1337,79 +1258,37 @@ void ExtentMap::load(const string& filename, bool fixFL) throw; } - // XXXPAT: Forcing the IDB path. Remove the fstream path once we see this works. - if (true || IDBPolicy::useHdfs()) + const char* filename_p = filename.c_str(); + scoped_ptr in(IDBDataFile::open( + IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), + filename_p, "r", 0)); + + if (!in) { - const char* filename_p = filename.c_str(); - scoped_ptr in(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "r", 0)); + log_errno("ExtentMap::load(): open"); + releaseFreeList(WRITE); + releaseEMEntryTable(WRITE); + throw ios_base::failure("ExtentMap::load(): open failed. Check the error log."); + } - if (!in) - { - log_errno("ExtentMap::load(): open"); - releaseFreeList(WRITE); - releaseEMEntryTable(WRITE); - throw ios_base::failure("ExtentMap::load(): open failed. Check the error log."); - } + try + { + int emVersion = 0; + int bytes = in->read((char*) &emVersion, sizeof(int)); - try + if (bytes == (int) sizeof(int) && emVersion == EM_MAGIC_V4) + loadVersion4(in.get()); + else { - int emVersion = 0; - int bytes = in->read((char*) &emVersion, sizeof(int)); - - if (bytes == (int) sizeof(int) && emVersion == EM_MAGIC_V4) - loadVersion4(in.get()); - else - { - log("ExtentMap::load(): That file is not a valid ExtentMap image"); - throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image"); - } - } - catch (...) - { - releaseFreeList(WRITE); - releaseEMEntryTable(WRITE); - throw; + log("ExtentMap::load(): That file is not a valid ExtentMap image"); + throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image"); } } - else + else // fstream path to be remove { - ifstream in; - in.open(filename.c_str(), ios_base::in | ios_base::binary); - - if (!in) - { - log_errno("ExtentMap::load(): open"); - releaseFreeList(WRITE); - releaseEMEntryTable(WRITE); - throw ios_base::failure("ExtentMap::load(): open failed. Check the error log."); - } - - in.exceptions(ios_base::badbit | ios_base::failbit); - - try - { - int emVersion; - in.read((char*) &emVersion, sizeof(int)); - - if (emVersion == EM_MAGIC_V4) - loadVersion4(in); - else - { - log("ExtentMap::load(): That file is not a valid ExtentMap image"); - throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image"); - } - } - catch (...) - { - in.close(); - releaseFreeList(WRITE); - releaseEMEntryTable(WRITE); - throw; - } - - in.close(); + releaseFreeList(WRITE); + releaseEMEntryTable(WRITE); + throw; } releaseFreeList(WRITE); @@ -1453,77 +1332,53 @@ void ExtentMap::save(const string& filename) throw runtime_error("ExtentMap::save(): got request to save an empty BRM"); } - // XXXPAT: I don't know why there are two options here. It can just use the IDBDataFile stuff. - // Forcing the IDB option to execute for now. Leaving the old fstream version there in case we find there's - // a case the IDB option doesn't work. - if (true || IDBPolicy::useHdfs()) + const char* filename_p = filename.c_str(); + scoped_ptr out(IDBDataFile::open( + IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), + filename_p, "wb", IDBDataFile::USE_VBUF)); + + if (!out) { - const char* filename_p = filename.c_str(); - scoped_ptr out(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "wb", IDBDataFile::USE_VBUF)); + log_errno("ExtentMap::save(): open"); + releaseFreeList(READ); + releaseEMEntryTable(READ); + throw ios_base::failure("ExtentMap::save(): open failed. Check the error log."); + } - if (!out) + loadSize[0] = EM_MAGIC_V4; + loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry); + loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries + + int bytes = 0; + + try + { + const int wsize = 3 * sizeof(int); + bytes = out->write((char*)loadSize, wsize); + + if (bytes != wsize) + throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); + } + catch (...) + { + releaseFreeList(READ); + releaseEMEntryTable(READ); + throw; + } + + allocdSize = fEMShminfo->allocdSize / sizeof(EMEntry); + //const int emEntrySize = sizeof(EMEntry); + + int first = -1, last = -1, err; + size_t progress, writeSize; + for (i = 0; i < allocdSize; i++) + { + if (fExtentMap[i].range.size > 0 && first == -1) + first = i; + else if (fExtentMap[i].range.size <= 0 && first != -1) { - log_errno("ExtentMap::save(): open"); - releaseFreeList(READ); - releaseEMEntryTable(READ); - throw ios_base::failure("ExtentMap::save(): open failed. Check the error log."); - } - - loadSize[0] = EM_MAGIC_V4; - loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry); - loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries - - int bytes = 0; - - try - { - const int wsize = 3 * sizeof(int); - bytes = out->write((char*)loadSize, wsize); - - if (bytes != wsize) - throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); - } - catch (...) - { - releaseFreeList(READ); - releaseEMEntryTable(READ); - throw; - } - - allocdSize = fEMShminfo->allocdSize / sizeof(EMEntry); - //const int emEntrySize = sizeof(EMEntry); - - int first = -1, last = -1, err; - size_t progress, writeSize; - for (i = 0; i < allocdSize; i++) - { - if (fExtentMap[i].range.size > 0 && first == -1) - first = i; - else if (fExtentMap[i].range.size <= 0 && first != -1) - { - last = i; - writeSize = (last - first) * sizeof(EMEntry); - progress = 0; - char *writePos = (char *) &fExtentMap[first]; - while (progress < writeSize) - { - err = out->write(writePos + progress, writeSize - progress); - if (err < 0) - { - releaseFreeList(READ); - releaseEMEntryTable(READ); - throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); - } - progress += err; - } - first = -1; - } - } - if (first != -1) - { - writeSize = (allocdSize - first) * sizeof(EMEntry); + last = i; + writeSize = (last - first) * sizeof(EMEntry); progress = 0; char *writePos = (char *) &fExtentMap[first]; while (progress < writeSize) @@ -1537,14 +1392,15 @@ void ExtentMap::save(const string& filename) } progress += err; } + first = -1; } - //allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - //const int inlineLbidRangeSize = sizeof(InlineLBIDRange); - + } + if (first != -1) + { + writeSize = (allocdSize - first) * sizeof(EMEntry); progress = 0; - writeSize = fFLShminfo->allocdSize; - char *writePos = (char *) fFreeList; + char *writePos = (char *) &fExtentMap[first]; while (progress < writeSize) { err = out->write(writePos + progress, writeSize - progress); @@ -1554,90 +1410,27 @@ void ExtentMap::save(const string& filename) releaseEMEntryTable(READ); throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); } - progress += err; } } - else // this is the fstream version to be expired + + //allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); + //const int inlineLbidRangeSize = sizeof(InlineLBIDRange); + + progress = 0; + writeSize = fFLShminfo->allocdSize; + char *writePos = (char *) fFreeList; + while (progress < writeSize) { - ofstream out; - - // Make em writes to disk use a buffer size of StrmBufSize bytes (instead of the default 8K) - const unsigned StrmBufSize = 1 * 1024 * 1024; - scoped_array buf(new char[StrmBufSize]); - out.rdbuf()->pubsetbuf(buf.get(), StrmBufSize); - - utmp = ::umask(0); - out.open(filename.c_str(), ios_base::out | ios_base::binary); - ::umask(utmp); - - if (!out) + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) { - log_errno("ExtentMap::save(): open"); releaseFreeList(READ); releaseEMEntryTable(READ); - throw ios_base::failure("ExtentMap::save(): open failed. Check the error log."); + throw ios_base::failure("ExtentMap::save(): write failed. Check the error log."); } - out.exceptions(ios_base::badbit); - - loadSize[0] = EM_MAGIC_V4; - loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry); - loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries - - try - { - out.write((char*)loadSize, 3 * sizeof(int)); - } - catch (...) - { - out.close(); - releaseFreeList(READ); - releaseEMEntryTable(READ); - throw; - } - - allocdSize = fEMShminfo->allocdSize / sizeof(EMEntry); - - for (i = 0; i < allocdSize; i++) - { - if (fExtentMap[i].range.size > 0) - { - try - { - out.write((char*) &fExtentMap[i], sizeof(EMEntry)); - } - catch (...) - { - out.close(); - releaseFreeList(READ); - releaseEMEntryTable(READ); - throw; - } - } - } - - allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); - - for (i = 0; i < allocdSize; i++) - { -// if (fFreeList[i].size > 0) { - try - { - out.write((char*) &fFreeList[i], sizeof(InlineLBIDRange)); - } - catch (...) - { - out.close(); - releaseFreeList(READ); - releaseEMEntryTable(READ); - throw; - } - -// } - } - - out.close(); + progress += err; } releaseFreeList(READ); diff --git a/versioning/BRM/oidserver.cpp b/versioning/BRM/oidserver.cpp index 6b8887012..d8184ed4e 100644 --- a/versioning/BRM/oidserver.cpp +++ b/versioning/BRM/oidserver.cpp @@ -117,34 +117,6 @@ namespace BRM boost::mutex OIDServer::fMutex; -#if 0 -void OIDServer::lockFile() const -{ - int err, errCount; - - int errnoSave = 0; - - for (errCount = 0, err = -1; err != 0 && errCount < MaxRetries;) - { - err = flock(fFd, LOCK_EX); - - if (err < 0 && errno != EINTR) // EINTR isn't really an error - { - errCount++; - errnoSave = errno; // save errno because perror may overwrite - perror("OIDServer::lockFile(): flock (retrying)"); - } - } - - if (errCount == MaxRetries) - { - ostringstream oss; - oss << "OIDServer::lockFile(): flock error: " << strerror(errnoSave); - throw ios_base::failure(oss.str()); - } -} -#endif - void OIDServer::writeData(uint8_t* buf, off_t offset, int size) const { int errCount, err, progress; @@ -153,73 +125,39 @@ void OIDServer::writeData(uint8_t* buf, off_t offset, int size) const if (size == 0) return; - // XXXPAT: Forcing the IDB* path. Get rid of the fstream path when appropriate. - if (true || IDBPolicy::useHdfs()) + for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++) { - for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++) - { - seekerr = fFp->seek(offset, SEEK_SET); + seekerr = fFp->seek(offset, SEEK_SET); - if (seekerr >= 0) - seekerr = fFp->tell(); // IDBDataFile may use fseek for seek. + if (seekerr >= 0) + seekerr = fFp->tell(); // IDBDataFile may use fseek for seek. - if (seekerr < 0) - perror("OIDServer::writeDataHdfs(): lseek"); - } - - if (errCount == MaxRetries) - throw ios_base::failure("OIDServer::writeDataHdfs(): lseek failed " - "too many times"); - - for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;) - { - err = fFp->write(&buf[progress], size - progress); - - if (err < 0) - { - if (errno != EINTR) // EINTR isn't really an error - { - errCount++; - perror("OIDServer::writeDataHdfs(): write (retrying)"); - } - } - else - progress += err; - } - - fFp->tell(); + if (seekerr < 0) + perror("OIDServer::writeData(): lseek"); } - else + + if (errCount == MaxRetries) + throw ios_base::failure("OIDServer::writeData(): lseek failed " + "too many times"); + + for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;) { - for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++) + err = fFp->write(&buf[progress], size - progress); + + if (err < 0) { - seekerr = lseek(fFd, offset, SEEK_SET); - - if (seekerr < 0) - perror("OIDServer::writeData(): lseek"); - } - - if (errCount == MaxRetries) - throw ios_base::failure("OIDServer::writeData(): lseek failed " - "too many times"); - - for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;) - { - err = write(fFd, &buf[progress], size - progress); - - if (err < 0) + if (errno != EINTR) // EINTR isn't really an error { - if (errno != EINTR) // EINTR isn't really an error - { - errCount++; - perror("OIDServer::writeData(): write (retrying)"); - } + errCount++; + perror("OIDServer::writeData(): write (retrying)"); } - else - progress += err; } + else + progress += err; } + fFp->tell(); + if (errCount == MaxRetries) throw ios_base::failure("OIDServer::writeData(): write error"); } @@ -232,73 +170,37 @@ void OIDServer::readData(uint8_t* buf, off_t offset, int size) const if (size == 0) return; - // XXXPAT: Forcing the IDB* path. Get rid of the fstream path when appropriate. - if (true || IDBPolicy::useHdfs()) + for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++) { - for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++) - { - seekerr = fFp->seek(offset, SEEK_SET); + seekerr = fFp->seek(offset, SEEK_SET); - if (seekerr >= 0) - seekerr = fFp->tell(); // IDBDataFile may use fseek for seek. + if (seekerr >= 0) + seekerr = fFp->tell(); // IDBDataFile may use fseek for seek. - if (seekerr < 0) - perror("OIDServer::readDataHdfs(): lseek"); - } - - if (errCount == MaxRetries) - throw ios_base::failure("OIDServer::readDataHdfs(): lseek failed " - "too many times"); - - for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;) - { - err = fFp->read(&buf[progress], size - progress); - - if (err < 0) - { - if (errno != EINTR) // EINTR isn't really an error - { - errCount++; - perror("OIDServer::readDataHdfs(): read (retrying)"); - } - } - else if (err == 0) - throw EOFException(); - else - progress += err; - } + if (seekerr < 0) + perror("OIDServer::readData(): lseek"); } - else + + if (errCount == MaxRetries) + throw ios_base::failure("OIDServer::readData(): lseek failed " + "too many times"); + + for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;) { - for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++) + err = fFp->read(&buf[progress], size - progress); + + if (err < 0) { - seekerr = lseek(fFd, offset, SEEK_SET); - - if (seekerr < 0) - perror("OIDServer::readData(): lseek"); - } - - if (errCount == MaxRetries) - throw ios_base::failure("OIDServer::readData(): lseek failed " - "too many times"); - - for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;) - { - err = read(fFd, &buf[progress], size - progress); - - if (err < 0) + if (errno != EINTR) // EINTR isn't really an error { - if (errno != EINTR) // EINTR isn't really an error - { - errCount++; - perror("OIDServer::readData(): read (retrying)"); - } + errCount++; + perror("OIDServer::readData(): read (retrying)"); } - else if (err == 0) - throw EOFException(); - else - progress += err; } + else if (err == 0) + throw EOFException(); + else + progress += err; } if (errCount == MaxRetries) @@ -349,14 +251,8 @@ void OIDServer::initializeBitmap() const } writeData(buf, 0, HeaderSize); - - // reset buf to all 0's and write the bitmap - //for (i = 0; i < HeaderSize; i++) - // buf[i] = 0; - - //for (i = 0; i < bitmapSize; i += HeaderSize) - // writeData(buf, HeaderSize + i, (bitmapSize - i > HeaderSize ? HeaderSize : bitmapSize - i)); + // write the new bitmap file uint8_t *bitmapbuf = new uint8_t[bitmapSize]; memset(bitmapbuf, 0, bitmapSize); writeData(bitmapbuf, HeaderSize, bitmapSize); @@ -389,118 +285,55 @@ OIDServer::OIDServer() : fFp(NULL), fFd(-1) throw runtime_error(os.str()); } - // XXXPAT: Forcing the IDB* path. - if (true || IDBPolicy::useHdfs()) + if (!IDBPolicy::exists(fFilename.c_str())) //no bitmap file { - if (!IDBPolicy::exists(fFilename.c_str())) //no bitmap file + BRM::DBRM em; + + if (!em.isEMEmpty()) { - BRM::DBRM em; + os << "Extent Map not empty and " << fFilename << " not found. Setting system to read-only"; + cerr << os.str() << endl; + log(os.str()); + em.setReadOnly(true); + throw runtime_error(os.str()); + } - if (!em.isEMEmpty()) - { - os << "Extent Map not empty and " << fFilename << " not found. Setting system to read-only"; - cerr << os.str() << endl; - log(os.str()); - em.setReadOnly(true); - throw runtime_error(os.str()); - } + fFp = IDBDataFile::open(IDBPolicy::getType(fFilename.c_str(), IDBPolicy::WRITEENG), + fFilename.c_str(), "w+b", 0, 1); - fFp = IDBDataFile::open(IDBPolicy::getType(fFilename.c_str(), IDBPolicy::WRITEENG), - fFilename.c_str(), "w+b", 0, 1); - - if (!fFp) - { - os << "Couldn't create oid bitmap file " << fFilename << ": " << - strerror(errno); - log(os.str()); - throw ios_base::failure(os.str()); - } + if (!fFp) + { + os << "Couldn't create oid bitmap file " << fFilename << ": " << + strerror(errno); + log(os.str()); + throw ios_base::failure(os.str()); + } #ifndef _MSC_VER - - //FIXME: - //fchmod(fFd, 0666); // XXXPAT: override umask at least for testing - if (fFp) - chmod(fFilename.c_str(), 0664); // XXXPAT: override umask at least for testing - + if (fFp) + chmod(fFilename.c_str(), 0664); // XXXPAT: override umask at least for testing #endif - try - { - initializeBitmap(); - } - catch (...) - { - delete fFp; - fFp = NULL; - throw; - } - } - else + try { - fFp = IDBDataFile::open(IDBPolicy::getType(fFilename.c_str(), IDBPolicy::WRITEENG), - fFilename.c_str(), "r+b", 0, 1); - - if (!fFp) - { - ostringstream os; - os << "Couldn't open oid bitmap file" << fFilename << ": " << - strerror(errno); - log(os.str()); - throw ios_base::failure(os.str()); - } + initializeBitmap(); + } + catch (...) + { + delete fFp; + fFp = NULL; + throw; } } else { - if (access(fFilename.c_str(), F_OK) != 0) //no bitmap file + fFp = IDBDataFile::open(IDBPolicy::getType(fFilename.c_str(), IDBPolicy::WRITEENG), + fFilename.c_str(), "r+b", 0, 1); + + if (!fFp) { - BRM::DBRM em; - - if (!em.isEMEmpty()) - { - os << "Extent Map not empty and " << fFilename << " not found. Setting system to read-only"; - cerr << os.str() << endl; - log(os.str()); - em.setReadOnly(true); - throw runtime_error(os.str()); - } - } - - fFd = open(fFilename.c_str(), O_CREAT | O_EXCL | O_RDWR | O_BINARY, 0664); - - if (fFd >= 0) - { -#ifndef _MSC_VER - //FIXME: - fchmod(fFd, 0666); // XXXPAT: override umask at least for testing -#endif - - try - { - initializeBitmap(); - } - catch (...) - { - close(fFd); - throw; - } - } - else if (errno == EEXIST) - { - fFd = open(fFilename.c_str(), O_RDWR | O_BINARY); - - if (fFd < 0) - { - os << "Couldn't open oid bitmap file " << fFilename << ": " << - strerror(errno); - log(os.str()); - throw ios_base::failure(os.str()); - } - } - else - { - os << "Couldn't create oid bitmap file " << fFilename << ": " << + ostringstream os; + os << "Couldn't open oid bitmap file" << fFilename << ": " << strerror(errno); log(os.str()); throw ios_base::failure(os.str()); @@ -622,8 +455,7 @@ retry: { writeData(buf, offset, byteSize); - if (true || IDBPolicy::useHdfs()) - fFp->flush(); + fFp->flush(); delete [] buf; return; @@ -669,8 +501,7 @@ retry: { writeData(buf, offset, byteSize); - if (true || IDBPolicy::useHdfs()) - fFp->flush(); + fFp->flush(); delete [] buf; return; @@ -790,9 +621,7 @@ void OIDServer::patchFreelist(struct FEntry* freelist, int start, int num) const if (changed) { writeData(reinterpret_cast(freelist), 0, HeaderSize); - - if (true || IDBPolicy::useHdfs()) - fFp->flush(); + fFp->flush(); } } @@ -819,8 +648,7 @@ int OIDServer::allocVBOID(uint16_t dbroot) throw; } - if (true || IDBPolicy::useHdfs()) - fFp->flush(); + fFp->flush(); return ret; } @@ -893,8 +721,7 @@ int OIDServer::allocOIDs(int num) writeData(reinterpret_cast(freelist), 0, HeaderSize); flipOIDBlock(bestMatchBegin, num, 0); - if (true || IDBPolicy::useHdfs()) - fFp->flush(); + fFp->flush(); return bestMatchBegin; } diff --git a/versioning/BRM/sessionmanagerserver.cpp b/versioning/BRM/sessionmanagerserver.cpp index 4c606e434..b98ca2d87 100644 --- a/versioning/BRM/sessionmanagerserver.cpp +++ b/versioning/BRM/sessionmanagerserver.cpp @@ -85,7 +85,7 @@ const uint32_t SessionManagerServer::SS_FORCE = 1 << 5; // In combination w const uint32_t SessionManagerServer::SS_QUERY_READY = 1 << 6; // Set by ProcManager when system is ready for queries -SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0), txnidfd(-1) +SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0) { config::Config* conf; string stmp; @@ -119,26 +119,6 @@ SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0), txnidfd txnidFilename = conf->getConfig("SessionManager", "TxnIDFile"); - if (false && !IDBPolicy::useHdfs()) - { - txnidfd = open(txnidFilename.c_str(), O_RDWR | O_CREAT | O_BINARY, 0664); - - if (txnidfd < 0) - { - perror("SessionManagerServer(): open"); - throw runtime_error("SessionManagerServer: Could not open the transaction ID file"); - } - - //FIXME: do we need this on Win? -#ifndef _MSC_VER - else - { - fchmod(txnidfd, 0664); - } - -#endif - } - semValue = maxTxns; _verID = 0; _sysCatVerID = 0; @@ -154,6 +134,9 @@ SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0), txnidfd } } +SessionManagerServer::~SessionManagerServer() +{ +} void SessionManagerServer::reset() { mutex.try_lock(); @@ -178,57 +161,7 @@ again: // If we fail to read a full four bytes for any value, then the // value isn't in the file, and we start with the default. - if (false && !IDBPolicy::useHdfs()) - { - // Last transaction id - lseek(txnidfd, 0, SEEK_SET); - err = read(txnidfd, &lastTxnID, 4); - - if (err < 0 && errno != EINTR) - { - perror("Sessionmanager::initSegment(): read"); - throw runtime_error("SessionManagerServer: read failed, aborting"); - } - else if (err < 0) - goto again; - else if (err == sizeof(int)) - _verID = lastTxnID; - - // last system catalog version id - err = read(txnidfd, &lastSysCatVerId, 4); - - if (err < 0 && errno != EINTR) - { - perror("Sessionmanager::initSegment(): read"); - throw runtime_error("SessionManagerServer: read failed, aborting"); - } - else if (err < 0) - goto again; - else if (err == sizeof(int)) - _sysCatVerID = lastSysCatVerId; - - // System state. Contains flags regarding the suspend state of the system. - err = read(txnidfd, &systemState, 4); - - if (err < 0 && errno == EINTR) - { - goto again; - } - else if (err == sizeof(int)) - { - // Turn off the pending and force flags. They make no sense for a clean start. - // Turn off the ready flag. DMLProc will set it back on when - // initialized. - systemState &= - ~(SS_READY | SS_QUERY_READY | SS_SUSPEND_PENDING | SS_SHUTDOWN_PENDING | SS_ROLLBACK | SS_FORCE); - } - else - { - // else no problem. System state wasn't saved. Might be an upgraded system. - systemState = 0; - } - } - else if (IDBPolicy::exists(txnidFilename.c_str())) + if (IDBPolicy::exists(txnidFilename.c_str())) { scoped_ptr txnidfp(IDBDataFile::open( IDBPolicy::getType(txnidFilename.c_str(), @@ -297,26 +230,7 @@ again: */ void SessionManagerServer::saveSystemState() { - if (false && !IDBPolicy::useHdfs()) - { - int err = 0; - uint32_t lSystemState = systemState; - - // We don't save the pending flags, the force flag or the ready flags. - lSystemState &= ~(SS_READY | SS_QUERY_READY | SS_SUSPEND_PENDING | SS_SHUTDOWN_PENDING | SS_FORCE); - lseek(txnidfd, 8, SEEK_SET); - err = write(txnidfd, &lSystemState, sizeof(int)); - - if (err < 0) - { - perror("SessionManagerServer::saveSystemState(): write(systemState)"); - throw runtime_error("SessionManagerServer::saveSystemState(): write(systemState) failed"); - } - } - else - { saveSMTxnIDAndState(); - } } const QueryContext SessionManagerServer::verID() @@ -377,25 +291,7 @@ const TxnID SessionManagerServer::newTxnID(const SID session, bool block, bool i if (isDDL) ++_sysCatVerID; - if (false && !IDBPolicy::useHdfs()) - { - int filedata[2]; - filedata[0] = _verID; - filedata[1] = _sysCatVerID; - - lseek(txnidfd, 0, SEEK_SET); - int err = write(txnidfd, filedata, 8); - - if (err < 0) - { - perror("SessionManagerServer::newTxnID(): write(verid)"); - throw runtime_error("SessionManagerServer::newTxnID(): write(verid) failed"); - } - } - else - { - saveSMTxnIDAndState(); - } + saveSMTxnIDAndState(); return ret; } diff --git a/versioning/BRM/sessionmanagerserver.h b/versioning/BRM/sessionmanagerserver.h index b25de41f9..e1782e3e8 100644 --- a/versioning/BRM/sessionmanagerserver.h +++ b/versioning/BRM/sessionmanagerserver.h @@ -123,10 +123,7 @@ public: * It does not destroy the semaphores. Those persist until the system * is shut down. */ - virtual ~SessionManagerServer() - { - if (txnidfd >= 0 ) close(txnidfd); - } + virtual ~SessionManagerServer(); /** @brief Gets the current version ID * @@ -276,7 +273,6 @@ private: int maxTxns; // the maximum number of concurrent transactions std::string txnidFilename; - int txnidfd; // file descriptor for the "last txnid" file execplan::CalpontSystemCatalog::SCN _verID; execplan::CalpontSystemCatalog::SCN _sysCatVerID; uint32_t systemState; diff --git a/versioning/BRM/slavecomm.cpp b/versioning/BRM/slavecomm.cpp index 61a9f1bb2..278b214f7 100644 --- a/versioning/BRM/slavecomm.cpp +++ b/versioning/BRM/slavecomm.cpp @@ -65,7 +65,7 @@ namespace BRM { SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) : - slave(s), currentSaveFD(-1), currentSaveFile(NULL), journalh(NULL) + slave(s), currentSaveFile(NULL), journalh(NULL) #ifdef _MSC_VER , fPids(0), fMaxPids(64) #endif @@ -138,21 +138,10 @@ SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) : firstSlave = true; journalName = savefile + "_journal"; const char* filename = journalName.c_str(); - uint32_t utmp = ::umask(0); - if (true || IDBPolicy::useHdfs()) - { - journalh = IDBDataFile::open( - IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "a", 0); - } - else - { - journal.open(filename, ios_base::binary | ios_base::out | ios_base::app); - } - - ::umask(utmp); - - if ((journal.is_open() == false) && (journalh == NULL)) + journalh = IDBDataFile::open( + IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "a", 0); + if (journalh == NULL) throw runtime_error("Could not open the BRM journal for writing!"); } else @@ -180,7 +169,7 @@ SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) : } SlaveComm::SlaveComm() - : currentSaveFD(-1), currentSaveFile(NULL), journalh(NULL) + : currentSaveFile(NULL), journalh(NULL) #ifdef _MSC_VER , fPids(0), fMaxPids(64) #endif @@ -222,7 +211,6 @@ SlaveComm::~SlaveComm() if (firstSlave) { - close(currentSaveFD); delete currentSaveFile; currentSaveFile = NULL; } @@ -2007,19 +1995,13 @@ void SlaveComm::do_confirm() if (firstSlave && (takeSnapshot || (journalCount >= snapshotInterval && snapshotInterval >= 0))) { - const char* filename = tmp.c_str(); - - if (false && !IDBPolicy::useHdfs() && currentSaveFD < 0) - { - currentSaveFD = open(filename, O_WRONLY | O_CREAT, 0664); - } - else if ((true || IDBPolicy::useHdfs()) && !currentSaveFile) + if (!currentSaveFile) { currentSaveFile = IDBDataFile::open( - IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "wb", 0); + IDBPolicy::getType(tmp.c_str(), IDBPolicy::WRITEENG), tmp.c_str(), "wb", 0); } - if (currentSaveFD < 0 && currentSaveFile == NULL) + if (currentSaveFile == NULL) { ostringstream os; os << "WorkerComm: failed to open the current savefile. errno: " @@ -2028,9 +2010,7 @@ void SlaveComm::do_confirm() throw runtime_error(os.str()); } -#ifndef _MSC_VER - chmod(filename, 0664); -#endif + tmp = savefile + (saveFileToggle ? 'A' : 'B'); slave->saveState(tmp); #ifndef _MSC_VER @@ -2038,74 +2018,33 @@ void SlaveComm::do_confirm() #endif int err = 0; - if (currentSaveFile) + // MCOL-1558. Make the _current file relative to DBRMRoot. + string relative = tmp.substr(tmp.find_last_of('/') + 1); + err = currentSaveFile->write(relative.c_str(), relative.length()); + + if (err < (int) relative.length()) { - // MCOL-1558. Make the _current file relative to DBRMRoot. - string relative = tmp.substr(tmp.find_last_of('/') + 1); - err = currentSaveFile->write(relative.c_str(), relative.length()); + ostringstream os; + os << "WorkerComm: currentfile write() returned " << err + << " file pointer is " << currentSaveFile; - if (err < (int) relative.length()) - { - ostringstream os; - os << "WorkerComm: currentfile write() returned " << err - << " file pointer is " << currentSaveFile; + if (err < 0) + os << " errno: " << strerror(errno); - if (err < 0) - os << " errno: " << strerror(errno); - - log(os.str()); - } - - currentSaveFile->flush(); - delete currentSaveFile; - currentSaveFile = NULL; - saveFileToggle = !saveFileToggle; - - const char* filename = journalName.c_str(); - //uint32_t utmp = ::umask(0); - delete journalh; - journalh = IDBDataFile::open( - IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "w+b", 0); - //::umask(utmp); - - if (!journalh) - throw runtime_error("Could not open the BRM journal for writing!"); + log(os.str()); } - else - { - lseek(currentSaveFD, 0, SEEK_SET); - // MCOL-1558. Make the _current file relative to DBRMRoot. - string relative = tmp.substr(tmp.find_last_of('/') + 1); - err = write(currentSaveFD, relative.c_str(), relative.length()); - if (err < (int) relative.length()) - { - ostringstream os; - os << "WorkerComm: currentfile write() returned " << err - << " fd is " << currentSaveFD; + currentSaveFile->flush(); + delete currentSaveFile; + currentSaveFile = NULL; + saveFileToggle = !saveFileToggle; - if (err < 0) - os << " errno: " << strerror(errno); + delete journalh; + journalh = IDBDataFile::open( + IDBPolicy::getType(journalName.c_str(), IDBPolicy::WRITEENG), journalName.c_str(), "w+b", 0); - log(os.str()); - } - -#ifdef _MSC_VER - //FIXME: Do we need to account for Windows EOL conversions? - _chsize_s(currentSaveFD, tmp.length()); - _commit(currentSaveFD); -#else - err = ftruncate(currentSaveFD, relative.length()); - fsync(currentSaveFD); -#endif - saveFileToggle = !saveFileToggle; - - /* Is there a nicer way to truncate the file using an ofstream? */ - journal.close(); - uint32_t utmp = ::umask(0); - journal.open(journalName.c_str(), ios_base::binary | ios_base::out | ios_base::trunc); - ::umask(utmp); - } + if (!journalh) + throw runtime_error("Could not open the BRM journal for writing!"); takeSnapshot = false; doSaveDelta = false; @@ -2258,70 +2197,29 @@ int SlaveComm::replayJournal(string prefix) const char* filename = journalName.c_str(); - if (true || IDBPolicy::useHdfs()) + IDBDataFile* journalf = IDBDataFile::open( + IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "rb", 0); + + if (!journalf) { - IDBDataFile* journalf = IDBDataFile::open( - IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "rb", 0); - - if (!journalf) - { - cout << "Error opening journal file " << fName << endl; - return -1; - } - - if (journalf->size() == 0) // empty file, nothing to replay - return 0; - - ssize_t readIn = 0; - - do - { - readIn = journalf->read((char*) &len, sizeof(len)); - - if (readIn > 0) - { - cmd.needAtLeast(len); - readIn = journalf->read((char*) cmd.getInputPtr(), len); - cmd.advanceInputPtr(len); - - try - { - processCommand(cmd); - } - catch (exception& e) - { - cout << e.what() << " Journal replay was incomplete." << endl; - slave->undoChanges(); - return -1; - } - - slave->confirmChanges(); - cmd.restart(); - ret++; - } - } - while (readIn > 0); + cout << "Error opening journal file " << fName << endl; + return -1; } - else + + if (journalf->size() == 0) // empty file, nothing to replay + return 0; + + ssize_t readIn = 0; + + do { - ifstream journalf; - journalf.open(filename, ios_base::in | ios_base::binary); + readIn = journalf->read((char*) &len, sizeof(len)); - if (!journalf.is_open()) + if (readIn > 0) { - cout << "Error opening journal file " << fName << endl; - return -1; - } - - while (journalf) - { - journalf.read((char*) &len, sizeof(len)); - - if (!journalf) - break; cmd.needAtLeast(len); - journalf.read((char*) cmd.getInputPtr(), len); + readIn = journalf->read((char*) cmd.getInputPtr(), len); cmd.advanceInputPtr(len); try @@ -2339,7 +2237,7 @@ int SlaveComm::replayJournal(string prefix) cmd.restart(); ret++; } - } + } while (readIn > 0); return ret; } @@ -2368,20 +2266,10 @@ void SlaveComm::saveDelta() { uint32_t len = delta.length(); - if (true || IDBPolicy::useHdfs()) - { - journalh->write((const char*) &len, sizeof(len)); - journalh->write((const char*) delta.buf(), delta.length()); - journalh->flush(); - } - else - { - journal.seekg(0, ios_base::end); - journal.write((const char*) &len, sizeof(len)); - journal.write((const char*) delta.buf(), delta.length()); - journal.sync(); - } - + + journalh->write((const char*) &len, sizeof(len)); + journalh->write((const char*) delta.buf(), delta.length()); + journalh->flush(); journalCount++; } catch (exception& e) diff --git a/versioning/BRM/slavecomm.h b/versioning/BRM/slavecomm.h index c4ba025dd..f967a2239 100644 --- a/versioning/BRM/slavecomm.h +++ b/versioning/BRM/slavecomm.h @@ -125,10 +125,8 @@ private: std::string savefile; bool release, die, firstSlave, saveFileToggle, takeSnapshot, doSaveDelta, standalone, printOnly; messageqcpp::ByteStream delta; - int currentSaveFD; idbdatafile::IDBDataFile* currentSaveFile; std::string journalName; - std::fstream journal; idbdatafile::IDBDataFile* journalh; int64_t snapshotInterval, journalCount; struct timespec MSG_TIMEOUT; diff --git a/versioning/BRM/tablelockserver.cpp b/versioning/BRM/tablelockserver.cpp index 8eb1676a7..291001b14 100644 --- a/versioning/BRM/tablelockserver.cpp +++ b/versioning/BRM/tablelockserver.cpp @@ -63,41 +63,21 @@ void TableLockServer::save() const char* filename_p = filename.c_str(); - if (true || IDBPolicy::useHdfs()) + scoped_ptr out(IDBDataFile::open( + IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), + filename_p, "wb", 0)); + + if (!out) + throw runtime_error("TableLockServer::save(): could not open save file"); + + out->write((char*) &count, 4); + + for (it = locks.begin(); it != locks.end(); ++it) { - scoped_ptr out(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "wb", 0)); - if (!out) - throw runtime_error("TableLockServer::save(): could not open save file"); + throw runtime_error("TableLockServer::save(): could not write save file"); - out->write((char*) &count, 4); - - for (it = locks.begin(); it != locks.end(); ++it) - { - if (!out) - throw runtime_error("TableLockServer::save(): could not write save file"); - - it->second.serialize(out.get()); - } - } - else - { - ofstream out(filename.c_str(), ios::trunc | ios::binary | ios::out ); - - if (!out) - throw runtime_error("TableLockServer::save(): could not open save file"); - - out.write((char*) &count, 4); - - for (it = locks.begin(); it != locks.end(); ++it) - { - if (!out) - throw runtime_error("TableLockServer::save(): could not write save file"); - - it->second.serialize(out); - } + it->second.serialize(out.get()); } } @@ -109,85 +89,42 @@ void TableLockServer::load() TableLockInfo tli; /* Need to standardize the file error handling */ - if (true || IDBPolicy::useHdfs()) + const char* filename_p = filename.c_str(); + scoped_ptr in(IDBDataFile::open( + IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), + filename_p, "rb", 0)); + + if (!in) { - const char* filename_p = filename.c_str(); - scoped_ptr in(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "rb", 0)); + ostringstream os; + os << "TableLockServer::load(): could not open the save file" + << filename; + log(os.str(), logging::LOG_TYPE_DEBUG); + return; + } - if (!in) + try + { + in->read((char*) &size, 4); + + for (i = 0; i < size; i++) { - ostringstream os; - os << "TableLockServer::load(): could not open the save file" - << filename; - log(os.str(), logging::LOG_TYPE_DEBUG); - return; - } + tli.deserialize(in.get()); + tli.id = sms->getUnique64(); // Need new #s... - try - { - in->read((char*) &size, 4); + if (tli.id == 0) // 0 is an error code + tli.id = sms->getUnique64(); - for (i = 0; i < size; i++) - { - tli.deserialize(in.get()); - tli.id = sms->getUnique64(); // Need new #s... - - if (tli.id == 0) // 0 is an error code - tli.id = sms->getUnique64(); - - locks[tli.id] = tli; - } - } - catch (std::exception& e) - { - ostringstream os; - os << "TableLockServer::load(): could not load save file " << filename << - " loaded " << i << "/" << size << " entries\n"; - log(os.str(), logging::LOG_TYPE_DEBUG); - throw; + locks[tli.id] = tli; } } - else + catch (std::exception& e) { - ifstream in(filename.c_str(), ios::binary | ios::in); - - if (!in) - { - ostringstream os; - os << "TableLockServer::load(): could not open the save file" - << filename; - log(os.str(), logging::LOG_TYPE_DEBUG); - return; - } - - in.exceptions(ios::failbit | ios::badbit); - - try - { - in.read((char*) &size, 4); - - for (i = 0; i < size; i++) - { - tli.deserialize(in); - tli.id = sms->getUnique64(); // Need new #s... - - if (tli.id == 0) // 0 is an error code - tli.id = sms->getUnique64(); - - locks[tli.id] = tli; - } - } - catch (std::exception& e) - { - ostringstream os; - - os << "TableLockServer::load(): could not load save file " << filename << - " loaded " << i << "/" << size << " entries\n"; - log(os.str(), logging::LOG_TYPE_DEBUG); - throw; - } + ostringstream os; + os << "TableLockServer::load(): could not load save file " << filename << + " loaded " << i << "/" << size << " entries\n"; + log(os.str(), logging::LOG_TYPE_DEBUG); + throw; } } diff --git a/versioning/BRM/vbbm.cpp b/versioning/BRM/vbbm.cpp index 02f070278..4e97102c0 100644 --- a/versioning/BRM/vbbm.cpp +++ b/versioning/BRM/vbbm.cpp @@ -21,7 +21,6 @@ ****************************************************************************/ #include -#include #include #include #include @@ -980,54 +979,6 @@ void VBBM::setReadOnly() * struct VBBMEntry * numEntries */ -void VBBM::loadVersion1(IDBDataFile* in) -{ - int vbbmEntries, i; - VBBMEntry entry; - - clear(); - - if (in->read((char*) &vbbmEntries, 4) != 4) - { - log_errno("VBBM::load()"); - throw runtime_error("VBBM::load(): Failed to read entry number"); - } - - for (i = 0; i < vbbmEntries; i++) - { - if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry)) - { - log_errno("VBBM::load()"); - throw runtime_error("VBBM::load(): Failed to load entry"); - } - - insert(entry.lbid, entry.verID, entry.vbOID, entry.vbFBO, true); - //confirmChanges(); - addVBFileIfNotExists(entry.vbOID); - } - - /* This will load the saved file data from 2.2, but it is not compatible with - * 3.0+. If enabled, take out the addVBFile..() call above - */ -#if 0 - int dummy, nFiles; - - in.read((char*) &nFiles, 4); - cout << "got nfiles = " << nFiles << endl; - in.read((char*) &dummy, 4); // an unused var in 3.0+ - - while (vbbm->nFiles < nFiles) - growVBBM(true); // this allocates one file, doesn't grow the main storage - - in.read((char*) files, sizeof(VBFileMetadata) * nFiles); - - for (i = 0; i < nFiles; i++) - cout << "file " << i << ": oid=" << files[i].OID << " size=" << files[i].fileSize - << " offset=" << files[i].nextOffset << endl; - -#endif - -} void VBBM::loadVersion2(IDBDataFile* in) { @@ -1091,19 +1042,6 @@ void VBBM::loadVersion2(IDBDataFile* in) for (i = 0; i < vbbmEntries; i++) insert(loadedEntries[i].lbid, loadedEntries[i].verID, loadedEntries[i].vbOID, loadedEntries[i].vbFBO, true); - - /* - for (i = 0; i < vbbmEntries; i++) - { - if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry)) - { - log_errno("VBBM::load()"); - throw runtime_error("VBBM::load(): Failed to load entry"); - } - - insert(entry.lbid, entry.verID, entry.vbOID, entry.vbFBO, true); - } - */ } @@ -1138,9 +1076,6 @@ void VBBM::load(string filename) switch (magic) { - case VBBM_MAGIC_V1: - loadVersion1(in.get()); - break; case VBBM_MAGIC_V2: loadVersion2(in.get()); @@ -1161,59 +1096,38 @@ void VBBM::save(string filename) int i; int var; - // XXXPAT: forcing the IDB* path. Delete the fstream path when appropriate. - if (true || IDBPolicy::useHdfs()) + const char* filename_p = filename.c_str(); + scoped_ptr out(IDBDataFile::open( + IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), + filename_p, "wb", IDBDataFile::USE_VBUF)); + + if (!out) { - const char* filename_p = filename.c_str(); - scoped_ptr out(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "wb", IDBDataFile::USE_VBUF)); + log_errno("VBBM::save()"); + throw runtime_error("VBBM::save(): Failed to open the file"); + } - if (!out) + var = VBBM_MAGIC_V2; + int bytesWritten = 0; + int bytesToWrite = 12; + bytesWritten += out->write((char*) &var, 4); + bytesWritten += out->write((char*) &vbbm->vbCurrentSize, 4); + bytesWritten += out->write((char*) &vbbm->nFiles, 4); + + bytesWritten += out->write((char*) files, sizeof(VBFileMetadata) * vbbm->nFiles); + bytesToWrite += sizeof(VBFileMetadata) * vbbm->nFiles; + + int first = -1, last = -1, err; + size_t progress, writeSize; + + for (i = 0; i < vbbm->vbCapacity; i++) + { + if (storage[i].lbid != -1 && first == -1) + first = i; + else if (storage[i].lbid == -1 && first != -1) { - log_errno("VBBM::save()"); - throw runtime_error("VBBM::save(): Failed to open the file"); - } - - var = VBBM_MAGIC_V2; - int bytesWritten = 0; - int bytesToWrite = 12; - bytesWritten += out->write((char*) &var, 4); - bytesWritten += out->write((char*) &vbbm->vbCurrentSize, 4); - bytesWritten += out->write((char*) &vbbm->nFiles, 4); - - bytesWritten += out->write((char*) files, sizeof(VBFileMetadata) * vbbm->nFiles); - bytesToWrite += sizeof(VBFileMetadata) * vbbm->nFiles; - - int first = -1, last = -1, err; - size_t progress, writeSize; - - for (i = 0; i < vbbm->vbCapacity; i++) - { - if (storage[i].lbid != -1 && first == -1) - first = i; - else if (storage[i].lbid == -1 && first != -1) - { - last = i; - writeSize = (last - first) * sizeof(VBBMEntry); - progress = 0; - char *writePos = (char *) &storage[first]; - while (progress < writeSize) - { - err = out->write(writePos + progress, writeSize - progress); - if (err < 0) - { - log_errno("VBBM::save()"); - throw runtime_error("VBBM::save(): Failed to write the file"); - } - progress += err; - } - first = -1; - } - } - if (first != -1) - { - writeSize = (vbbm->vbCapacity - first) * sizeof(VBBMEntry); + last = i; + writeSize = (last - first) * sizeof(VBBMEntry); progress = 0; char *writePos = (char *) &storage[first]; while (progress < writeSize) @@ -1226,49 +1140,27 @@ void VBBM::save(string filename) } progress += err; } + first = -1; } - /* - for (i = 0; i < vbbm->vbCapacity; i++) - { - if (storage[i].lbid != -1) - { - bytesToWrite += sizeof(VBBMEntry); - bytesWritten += out->write((char*)&storage[i], sizeof(VBBMEntry)); - } - } - - if (bytesWritten != bytesToWrite) - { - log_errno("VBBM::save()"); - throw runtime_error("VBBM::save(): Failed to write the file"); - } - */ } - else + if (first != -1) { - ofstream out; - out.open(filename.c_str(), ios_base::trunc | ios_base::out | ios_base::binary); - - if (!out) + writeSize = (vbbm->vbCapacity - first) * sizeof(VBBMEntry); + progress = 0; + char *writePos = (char *) &storage[first]; + while (progress < writeSize) { - log_errno("VBBM::save()"); - throw runtime_error("VBBM::save(): Failed to open the file"); + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) + { + log_errno("VBBM::save()"); + throw runtime_error("VBBM::save(): Failed to write the file"); + } + progress += err; } - - out.exceptions(ios_base::badbit); - - var = VBBM_MAGIC_V2; - out.write((char*) &var, 4); - out.write((char*) &vbbm->vbCurrentSize, 4); - out.write((char*) &vbbm->nFiles, 4); - - out.write((char*) files, sizeof(VBFileMetadata) * vbbm->nFiles); - - for (i = 0; i < vbbm->vbCapacity; i++) - if (storage[i].lbid != -1) - out.write((char*)&storage[i], sizeof(VBBMEntry)); } + #if 0 cout << "saving... nfiles=" << vbbm->nFiles << "\n"; diff --git a/versioning/BRM/vbbm.h b/versioning/BRM/vbbm.h index e8b9c8c49..b2e430d3e 100644 --- a/versioning/BRM/vbbm.h +++ b/versioning/BRM/vbbm.h @@ -214,7 +214,6 @@ public: EXPORT void clear(); EXPORT void load(std::string filename); - EXPORT void loadVersion1(idbdatafile::IDBDataFile* in); EXPORT void loadVersion2(idbdatafile::IDBDataFile* in); EXPORT void save(std::string filename); diff --git a/versioning/BRM/vss.cpp b/versioning/BRM/vss.cpp index d75034da2..c225560a3 100644 --- a/versioning/BRM/vss.cpp +++ b/versioning/BRM/vss.cpp @@ -21,7 +21,6 @@ ****************************************************************************/ #include -#include #include //#define NDEBUG #include @@ -1294,57 +1293,36 @@ void VSS::save(string filename) int i; struct Header header; - // XXXPAT: Forcing the IDB* path to run. Delete the fstream path when appropriate. - if (true || IDBPolicy::useHdfs()) + const char* filename_p = filename.c_str(); + scoped_ptr out(IDBDataFile::open( + IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), + filename_p, "wb", IDBDataFile::USE_VBUF)); + + if (!out) { - const char* filename_p = filename.c_str(); - scoped_ptr out(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "wb", IDBDataFile::USE_VBUF)); + log_errno("VSS::save()"); + throw runtime_error("VSS::save(): Failed to open the file"); + } - if (!out) - { - log_errno("VSS::save()"); - throw runtime_error("VSS::save(): Failed to open the file"); - } + header.magic = VSS_MAGIC_V1; + header.entries = vss->currentSize; - header.magic = VSS_MAGIC_V1; - header.entries = vss->currentSize; + if (out->write((char*)&header, sizeof(header)) != sizeof(header)) + { + log_errno("VSS::save()"); + throw runtime_error("VSS::save(): Failed to write header to the file"); + } - if (out->write((char*)&header, sizeof(header)) != sizeof(header)) + int first = -1, last = -1, err; + size_t progress, writeSize; + for (i = 0; i < vss->capacity; i++) + { + if (storage[i].lbid != -1 && first == -1) + first = i; + else if (storage[i].lbid == -1 && first != -1) { - log_errno("VSS::save()"); - throw runtime_error("VSS::save(): Failed to write header to the file"); - } - - int first = -1, last = -1, err; - size_t progress, writeSize; - for (i = 0; i < vss->capacity; i++) - { - if (storage[i].lbid != -1 && first == -1) - first = i; - else if (storage[i].lbid == -1 && first != -1) - { - last = i; - writeSize = (last - first) * sizeof(VSSEntry); - progress = 0; - char *writePos = (char *) &storage[first]; - while (progress < writeSize) - { - err = out->write(writePos + progress, writeSize - progress); - if (err < 0) - { - log_errno("VSS::save()"); - throw runtime_error("VSS::save(): Failed to write the file"); - } - progress += err; - } - first = -1; - } - } - if (first != -1) - { - writeSize = (vss->capacity - first) * sizeof(VSSEntry); + last = i; + writeSize = (last - first) * sizeof(VSSEntry); progress = 0; char *writePos = (char *) &storage[first]; while (progress < writeSize) @@ -1357,54 +1335,24 @@ void VSS::save(string filename) } progress += err; } + first = -1; } - - /* - for (i = 0; i < vss->capacity; i++) - { - if (storage[i].lbid != -1) - { - if (out->write((char*)&storage[i], sizeof(VSSEntry)) != sizeof(VSSEntry)) - { - log_errno("VSS::save()"); - throw runtime_error("VSS::save(): Failed to write vss entry to the file"); - } - } - } - */ } - else + if (first != -1) { - ofstream out; - out.open(filename.c_str(), ios_base::trunc | ios_base::out | ios_base::binary); - //::umask(utmp); - - if (!out) + writeSize = (vss->capacity - first) * sizeof(VSSEntry); + progress = 0; + char *writePos = (char *) &storage[first]; + while (progress < writeSize) { - log_errno("VSS::save()"); - throw runtime_error("VSS::save(): Failed to open the file"); + err = out->write(writePos + progress, writeSize - progress); + if (err < 0) + { + log_errno("VSS::save()"); + throw runtime_error("VSS::save(): Failed to write the file"); + } + progress += err; } - - out.exceptions(ios_base::badbit); - - header.magic = VSS_MAGIC_V1; - header.entries = vss->currentSize; - - try - { - out.write((char*)&header, sizeof(header)); - - for (i = 0; i < vss->capacity; i++) - if (storage[i].lbid != -1) - out.write((char*)&storage[i], sizeof(VSSEntry)); - } - catch (std::exception& e) - { - out.close(); - throw; - } - - out.close(); } } @@ -1517,94 +1465,11 @@ void VSS::load(string filename) for (i = 0; i < header.entries; i++) insert(loadedEntries[i].lbid, loadedEntries[i].verID, loadedEntries[i].vbFlag, loadedEntries[i].locked, true); - - /* - for (i = 0; i < header.entries; i++) - { - if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry)) - { - log_errno("VSS::load()"); - throw runtime_error("VSS::load(): Failed to read entry"); - } - - insert(entry.lbid, entry.verID, entry.vbFlag, entry.locked, true); - } - - */ //time2 = microsec_clock::local_time(); //cout << "done loading " << time2 << " duration: " << time2-time1 << endl; } -#ifndef __LP64__ -//This code is OBE now that the structs are padded correctly -struct VSSEntry_ -{ - LBID_t lbid; - VER_t verID; - bool vbFlag; - bool locked; - int next; - uint32_t pad1; -}; - -void VSS::load64(string filename) -{ - int i; - struct Header header; - struct VSSEntry_ entry; - - const char* filename_p = filename.c_str(); - scoped_ptr in(IDBDataFile::open( - IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG), - filename_p, "rb", 0)); - - if (!in) - { - log_errno("VSS::load()"); - throw runtime_error("VSS::load(): Failed to open the file"); - } - - if (in->read((char*)&header, sizeof(header)) != sizeof(header)) - { - log_errno("VSS::load()"); - throw runtime_error("VSS::load(): Failed to read header"); - } - - if (header.magic != VSS_MAGIC_V1) - { - log("VSS::load(): Bad magic. Not a VSS file?"); - throw runtime_error("VSS::load(): Bad magic. Not a VSS file?"); - } - - if (header.entries < 0) - { - log("VSS::load(): Bad size. Not a VSS file?"); - throw runtime_error("VSS::load(): Bad size. Not a VSS file?"); - } - - for (i = 0; i < vss->capacity; i++) - storage[i].lbid = -1; - - for (i = 0; i < vss->numHashBuckets; i++) - hashBuckets[i] = -1; - - vss->currentSize = 0; - vss->lockedEntryCount = 0; - vss->LWM = 0; - - for (i = 0; i < header.entries; i++) - { - if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry)) - { - log_errno("VSS::load()"); - throw runtime_error("VSS::load(): Failed to read entry"); - } - - insert(entry.lbid, entry.verID, entry.vbFlag, entry.locked, true); - } -} -#endif #ifdef BRM_DEBUG // read lock diff --git a/versioning/BRM/vss.h b/versioning/BRM/vss.h index b3e20732b..9595b0d26 100644 --- a/versioning/BRM/vss.h +++ b/versioning/BRM/vss.h @@ -250,10 +250,6 @@ public: EXPORT void clear(); EXPORT void load(std::string filename); -#ifndef __LP64__ - //This method is OBE now that the structs are padded correctly - EXPORT void load64(std::string filename); -#endif EXPORT void save(std::string filename); #ifdef BRM_DEBUG