Merge branch 'develop' into S3-project

oamapps/postConfigure/installer.cpp

@@ -989,7 +989,7 @@ int main(int argc, char* argv[])
         system(cmd.c_str());
 
         if (oam.checkLogStatus(logFile, "System Catalog created") )
-            cout << endl << "System Catalog Successfull Created" << endl;
+            cout << endl << "System Catalog Successfully Created" << endl;
         else
         {
             if ( oam.checkLogStatus(logFile, "System catalog appears to exist") )

oamapps/postConfigure/postConfigure.cpp

@@ -808,6 +808,7 @@ int main(int argc, char* argv[])
         try
         {
             sysConfig->setConfig(InstallSection, "MySQLPort", mysqlPort);
+            sysConfig->setConfig("CrossEngineSupport", "Port", mysqlPort);
         }
         catch (...)
         {}

oamapps/postConfigure/quick_installer_single_server.sh

@@ -6,13 +6,13 @@
 
 for arg in "$@"; do
 	if [ `expr -- "$arg" : '--help'` -eq 6 ]; then
-		echo "Usage ./quick_installer_multi_server.sh"
+		echo "Usage ./quick_installer_single_server.sh"
 		echo ""
 		echo "Quick Installer for a Single Server MariaDB ColumnStore Install"
 		echo ""
 		exit 1
 	else
-		echo "quick_installer_multi_server.sh: ignoring unknown argument: $arg" 1>&2
+		echo "quick_installer_single_server.sh: ignoring unknown argument: $arg" 1>&2
 	fi
 done
 

versioning/BRM/blockresolutionmanager.cpp

@@ -109,20 +109,9 @@ int BlockResolutionManager::saveState(string filename) throw()
         // truncate teh file if already exists since no truncate in HDFS.
         const char* filename = journalFilename.c_str();
 
-        if (true || IDBPolicy::useHdfs())
+        IDBDataFile* journal = IDBDataFile::open(
-        {
+                                   IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "wb", 0);
-            IDBDataFile* journal = IDBDataFile::open(
+        delete journal;
-                                       IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "wb", 0);
-            delete journal;
-        }
-        else
-        {
-            ofstream journal;
-            uint32_t utmp = ::umask(0);
-            journal.open(filename, ios_base::out | ios_base::trunc | ios_base::binary);
-            journal.close();
-            ::umask(utmp);
-        }
 
         vbbm.save(vbbmFilename);
         vss.save(vssFilename);

versioning/BRM/extentmap.cpp

@@ -1133,84 +1133,6 @@ void ExtentMap::reserveLBIDRange(LBID_t start, uint8_t size)
 	    ...   (* numFL)
 */
 
-void ExtentMap::loadVersion4(ifstream& in)
-{
-    int emNumElements, flNumElements;
-
-    in.read((char*) &emNumElements, sizeof(int));
-    in.read((char*) &flNumElements, sizeof(int));
-    idbassert(emNumElements > 0);
-
-    void *fExtentMapPtr = static_cast<void*>(fExtentMap);
-    memset(fExtentMapPtr, 0, fEMShminfo->allocdSize);
-    fEMShminfo->currentSize = 0;
-
-    // init the free list
-    memset(fFreeList, 0, fFLShminfo->allocdSize);
-    fFreeList[0].size = (1 << 26);   // 2^36 LBIDs
-    fFLShminfo->currentSize = sizeof(InlineLBIDRange);
-
-    // @Bug 3498
-    // Calculate how big an extent map we're going to need and allocate it in one call
-    if ((fEMShminfo->allocdSize / sizeof(EMEntry)) < (unsigned)emNumElements)
-    {
-        size_t nrows = emNumElements;
-
-        //Round up to the nearest EM_INCREMENT_ROWS
-        if ((nrows % EM_INCREMENT_ROWS) != 0)
-        {
-            nrows /= EM_INCREMENT_ROWS;
-            nrows++;
-            nrows *= EM_INCREMENT_ROWS;
-        }
-
-        growEMShmseg(nrows);
-    }
-
-    for (int i = 0; i < emNumElements; i++)
-    {
-        in.read((char*) &fExtentMap[i], sizeof(EMEntry));
-        reserveLBIDRange(fExtentMap[i].range.start, fExtentMap[i].range.size);
-
-        //@bug 1911 - verify status value is valid
-        if (fExtentMap[i].status < EXTENTSTATUSMIN ||
-                fExtentMap[i].status > EXTENTSTATUSMAX)
-            fExtentMap[i].status = EXTENTAVAILABLE;
-    }
-
-    fEMShminfo->currentSize = emNumElements * sizeof(EMEntry);
-
-#ifdef DUMP_EXTENT_MAP
-    EMEntry* emSrc = fExtentMap;
-    cout << "lbid\tsz\toid\tfbo\thwm\tpart#\tseg#\tDBRoot\twid\tst\thi\tlo\tsq\tv" << endl;
-
-    for (int i = 0; i < emNumElements; i++)
-    {
-        cout <<
-             emSrc[i].start
-             << '\t' << emSrc[i].size
-             << '\t' << emSrc[i].fileID
-             << '\t' << emSrc[i].blockOffset
-             << '\t' << emSrc[i].HWM
-             << '\t' << emSrc[i].partitionNum
-             << '\t' << emSrc[i].segmentNum
-             << '\t' << emSrc[i].dbRoot
-             << '\t' << emSrc[i].status
-             << '\t' << emSrc[i].partition.cprange.hi_val
-             << '\t' << emSrc[i].partition.cprange.lo_val
-             << '\t' << emSrc[i].partition.cprange.sequenceNum
-             << '\t' << (int)(emSrc[i].partition.cprange.isValid)
-             << endl;
-    }
-
-    cout << "Free list entries:" << endl;
-    cout << "start\tsize" << endl;
-
-    for (int i = 0; i < flNumElements; i++)
-        cout << fFreeList[i].start << '\t' << fFreeList[i].size << endl;
-
-#endif
-}
 
 void ExtentMap::loadVersion4(IDBDataFile* in)
 {
@@ -1264,7 +1186,6 @@ void ExtentMap::loadVersion4(IDBDataFile* in)
             log_errno("ExtentMap::loadVersion4(): read ");
             throw runtime_error("ExtentMap::loadVersion4(): read failed. Check the error log.");
         }
-
         progress += (uint) err;
     }
     
@@ -1337,79 +1258,37 @@ void ExtentMap::load(const string& filename, bool fixFL)
         throw;
     }
 
-    // XXXPAT: Forcing the IDB path.  Remove the fstream path once we see this works.
+    const char* filename_p = filename.c_str();
-    if (true || IDBPolicy::useHdfs())
+    scoped_ptr<IDBDataFile>  in(IDBDataFile::open(
+                                    IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
+                                    filename_p, "r", 0));
+
+    if (!in)
     {
-        const char* filename_p = filename.c_str();
+        log_errno("ExtentMap::load(): open");
-        scoped_ptr<IDBDataFile>  in(IDBDataFile::open(
+        releaseFreeList(WRITE);
-                                        IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
+        releaseEMEntryTable(WRITE);
-                                        filename_p, "r", 0));
+        throw ios_base::failure("ExtentMap::load(): open failed. Check the error log.");
+    }
 
-        if (!in)
+    try
-        {
+    {
-            log_errno("ExtentMap::load(): open");
+        int emVersion = 0;
-            releaseFreeList(WRITE);
+        int bytes = in->read((char*) &emVersion, sizeof(int));
-            releaseEMEntryTable(WRITE);
-            throw ios_base::failure("ExtentMap::load(): open failed. Check the error log.");
-        }
 
-        try
+        if (bytes == (int) sizeof(int) && emVersion == EM_MAGIC_V4)
+            loadVersion4(in.get());
+        else
         {
-            int emVersion = 0;
+            log("ExtentMap::load(): That file is not a valid ExtentMap image");
-            int bytes = in->read((char*) &emVersion, sizeof(int));
+            throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image");
-
-            if (bytes == (int) sizeof(int) && emVersion == EM_MAGIC_V4)
-                loadVersion4(in.get());
-            else
-            {
-                log("ExtentMap::load(): That file is not a valid ExtentMap image");
-                throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image");
-            }
-        }
-        catch (...)
-        {
-            releaseFreeList(WRITE);
-            releaseEMEntryTable(WRITE);
-            throw;
         }
     }
-    else
+    else   // fstream path to be remove
     {
-        ifstream in;
+        releaseFreeList(WRITE);
-        in.open(filename.c_str(), ios_base::in | ios_base::binary);
+        releaseEMEntryTable(WRITE);
-
+        throw;
-        if (!in)
-        {
-            log_errno("ExtentMap::load(): open");
-            releaseFreeList(WRITE);
-            releaseEMEntryTable(WRITE);
-            throw ios_base::failure("ExtentMap::load(): open failed. Check the error log.");
-        }
-
-        in.exceptions(ios_base::badbit | ios_base::failbit);
-
-        try
-        {
-            int emVersion;
-            in.read((char*) &emVersion, sizeof(int));
-
-            if (emVersion == EM_MAGIC_V4)
-                loadVersion4(in);
-            else
-            {
-                log("ExtentMap::load(): That file is not a valid ExtentMap image");
-                throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image");
-            }
-        }
-        catch (...)
-        {
-            in.close();
-            releaseFreeList(WRITE);
-            releaseEMEntryTable(WRITE);
-            throw;
-        }
-
-        in.close();
     }
 
     releaseFreeList(WRITE);
@@ -1453,77 +1332,53 @@ void ExtentMap::save(const string& filename)
         throw runtime_error("ExtentMap::save(): got request to save an empty BRM");
     }
 
-    // XXXPAT: I don't know why there are two options here.  It can just use the IDBDataFile stuff.
+    const char* filename_p = filename.c_str();
-    // Forcing the IDB option to execute for now.  Leaving the old fstream version there in case we find there's 
+    scoped_ptr<IDBDataFile> out(IDBDataFile::open(
-    // a case the IDB option doesn't work.
+                                    IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
-    if (true || IDBPolicy::useHdfs())
+                                    filename_p, "wb", IDBDataFile::USE_VBUF));
+
+    if (!out)
     {
-        const char* filename_p = filename.c_str();
+        log_errno("ExtentMap::save(): open");
-        scoped_ptr<IDBDataFile> out(IDBDataFile::open(
+        releaseFreeList(READ);
-                                        IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
+        releaseEMEntryTable(READ);
-                                        filename_p, "wb", IDBDataFile::USE_VBUF));
+        throw ios_base::failure("ExtentMap::save(): open failed. Check the error log.");
+    }
 
-        if (!out)
+    loadSize[0] = EM_MAGIC_V4;
+    loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry);
+    loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries
+
+    int bytes = 0;
+
+    try
+    {
+        const int wsize = 3 * sizeof(int);
+        bytes = out->write((char*)loadSize, wsize);
+
+        if (bytes != wsize)
+            throw ios_base::failure("ExtentMap::save(): write failed. Check the error log.");
+    }
+    catch (...)
+    {
+        releaseFreeList(READ);
+        releaseEMEntryTable(READ);
+        throw;
+    }
+
+    allocdSize = fEMShminfo->allocdSize / sizeof(EMEntry);
+    //const int emEntrySize = sizeof(EMEntry);
+
+    int first = -1, last = -1, err;
+    size_t progress, writeSize;
+    for (i = 0; i < allocdSize; i++)
+    {
+        if (fExtentMap[i].range.size > 0 && first == -1)
+            first = i;
+        else if (fExtentMap[i].range.size <= 0 && first != -1)
         {
-            log_errno("ExtentMap::save(): open");
+            last = i;
-            releaseFreeList(READ);
+            writeSize = (last - first) * sizeof(EMEntry);
-            releaseEMEntryTable(READ);
-            throw ios_base::failure("ExtentMap::save(): open failed. Check the error log.");
-        }
-
-        loadSize[0] = EM_MAGIC_V4;
-        loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry);
-        loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries
-
-        int bytes = 0;
-
-        try
-        {
-            const int wsize = 3 * sizeof(int);
-            bytes = out->write((char*)loadSize, wsize);
-
-            if (bytes != wsize)
-                throw ios_base::failure("ExtentMap::save(): write failed. Check the error log.");
-        }
-        catch (...)
-        {
-            releaseFreeList(READ);
-            releaseEMEntryTable(READ);
-            throw;
-        }
-
-        allocdSize = fEMShminfo->allocdSize / sizeof(EMEntry);
-        //const int emEntrySize = sizeof(EMEntry);
-
-        int first = -1, last = -1, err;
-        size_t progress, writeSize;
-        for (i = 0; i < allocdSize; i++)
-        {
-            if (fExtentMap[i].range.size > 0 && first == -1)
-                first = i;
-            else if (fExtentMap[i].range.size <= 0 && first != -1)
-            {
-                last = i;
-                writeSize = (last - first) * sizeof(EMEntry);
-                progress = 0;
-                char *writePos = (char *) &fExtentMap[first];
-                while (progress < writeSize)
-                {
-                    err = out->write(writePos + progress, writeSize - progress);
-                    if (err < 0)
-                    {
-                        releaseFreeList(READ);
-                        releaseEMEntryTable(READ);
-                        throw ios_base::failure("ExtentMap::save(): write failed. Check the error log.");
-                    }
-                    progress += err;
-                }
-                first = -1;
-            }
-        }
-        if (first != -1)
-        {
-            writeSize = (allocdSize - first) * sizeof(EMEntry);
             progress = 0;
             char *writePos = (char *) &fExtentMap[first];
             while (progress < writeSize)
@@ -1537,14 +1392,15 @@ void ExtentMap::save(const string& filename)
                 }
                 progress += err;
             }
+            first = -1;
         }
 
-        //allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange);
+    }
-        //const int inlineLbidRangeSize = sizeof(InlineLBIDRange);
+    if (first != -1)
-
+    {
+        writeSize = (allocdSize - first) * sizeof(EMEntry);
         progress = 0;
-        writeSize = fFLShminfo->allocdSize;
+        char *writePos = (char *) &fExtentMap[first];
-        char *writePos = (char *) fFreeList;
         while (progress < writeSize)
         {
             err = out->write(writePos + progress, writeSize - progress);
@@ -1554,90 +1410,27 @@ void ExtentMap::save(const string& filename)
                 releaseEMEntryTable(READ);
                 throw ios_base::failure("ExtentMap::save(): write failed. Check the error log.");
             }
-
             progress += err;
         }
     }
-    else   // this is the fstream version to be expired
+
+    //allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange);
+    //const int inlineLbidRangeSize = sizeof(InlineLBIDRange);
+
+    progress = 0;
+    writeSize = fFLShminfo->allocdSize;
+    char *writePos = (char *) fFreeList;
+    while (progress < writeSize)
     {
-        ofstream out;
+        err = out->write(writePos + progress, writeSize - progress);
-
+        if (err < 0)
-        // Make em writes to disk use a buffer size of StrmBufSize bytes (instead of the default 8K)
-        const unsigned StrmBufSize = 1 * 1024 * 1024;
-        scoped_array<char> buf(new char[StrmBufSize]);
-        out.rdbuf()->pubsetbuf(buf.get(), StrmBufSize);
-
-        utmp = ::umask(0);
-        out.open(filename.c_str(), ios_base::out | ios_base::binary);
-        ::umask(utmp);
-
-        if (!out)
         {
-            log_errno("ExtentMap::save(): open");
             releaseFreeList(READ);
             releaseEMEntryTable(READ);
-            throw ios_base::failure("ExtentMap::save(): open failed. Check the error log.");
+            throw ios_base::failure("ExtentMap::save(): write failed. Check the error log.");
         }
 
-        out.exceptions(ios_base::badbit);
+        progress += err;
-
-        loadSize[0] = EM_MAGIC_V4;
-        loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry);
-        loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries
-
-        try
-        {
-            out.write((char*)loadSize, 3 * sizeof(int));
-        }
-        catch (...)
-        {
-            out.close();
-            releaseFreeList(READ);
-            releaseEMEntryTable(READ);
-            throw;
-        }
-
-        allocdSize = fEMShminfo->allocdSize / sizeof(EMEntry);
-
-        for (i = 0; i < allocdSize; i++)
-        {
-            if (fExtentMap[i].range.size > 0)
-            {
-                try
-                {
-                    out.write((char*) &fExtentMap[i], sizeof(EMEntry));
-                }
-                catch (...)
-                {
-                    out.close();
-                    releaseFreeList(READ);
-                    releaseEMEntryTable(READ);
-                    throw;
-                }
-            }
-        }
-
-        allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange);
-
-        for (i = 0; i < allocdSize; i++)
-        {
-//			if (fFreeList[i].size > 0) {
-            try
-            {
-                out.write((char*) &fFreeList[i], sizeof(InlineLBIDRange));
-            }
-            catch (...)
-            {
-                out.close();
-                releaseFreeList(READ);
-                releaseEMEntryTable(READ);
-                throw;
-            }
-
-//			}
-        }
-
-        out.close();
     }
 
     releaseFreeList(READ);

versioning/BRM/oidserver.cpp

@@ -117,34 +117,6 @@ namespace BRM
 
 boost::mutex OIDServer::fMutex;
 
-#if 0
-void OIDServer::lockFile() const
-{
-    int err, errCount;
-
-    int errnoSave = 0;
-
-    for (errCount = 0, err = -1; err != 0 && errCount < MaxRetries;)
-    {
-        err = flock(fFd, LOCK_EX);
-
-        if (err < 0 && errno != EINTR)    // EINTR isn't really an error
-        {
-            errCount++;
-            errnoSave = errno; // save errno because perror may overwrite
-            perror("OIDServer::lockFile(): flock (retrying)");
-        }
-    }
-
-    if (errCount == MaxRetries)
-    {
-        ostringstream oss;
-        oss << "OIDServer::lockFile(): flock error:  " << strerror(errnoSave);
-        throw ios_base::failure(oss.str());
-    }
-}
-#endif
-
 void OIDServer::writeData(uint8_t* buf, off_t offset, int size) const
 {
     int errCount, err, progress;
@@ -153,73 +125,39 @@ void OIDServer::writeData(uint8_t* buf, off_t offset, int size) const
     if (size == 0)
         return;
 
-    // XXXPAT: Forcing the IDB* path.  Get rid of the fstream path when appropriate.
+    for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++)
-    if (true || IDBPolicy::useHdfs())
     {
-        for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++)
+        seekerr = fFp->seek(offset, SEEK_SET);
-        {
-            seekerr = fFp->seek(offset, SEEK_SET);
 
-            if (seekerr >= 0)
+        if (seekerr >= 0)
-                seekerr = fFp->tell(); // IDBDataFile may use fseek for seek.
+            seekerr = fFp->tell(); // IDBDataFile may use fseek for seek.
 
-            if (seekerr < 0)
+        if (seekerr < 0)
-                perror("OIDServer::writeDataHdfs(): lseek");
+            perror("OIDServer::writeData(): lseek");
-        }
-
-        if (errCount == MaxRetries)
-            throw ios_base::failure("OIDServer::writeDataHdfs(): lseek failed "
-                                    "too many times");
-
-        for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;)
-        {
-            err = fFp->write(&buf[progress], size - progress);
-
-            if (err < 0)
-            {
-                if (errno != EINTR)    // EINTR isn't really an error
-                {
-                    errCount++;
-                    perror("OIDServer::writeDataHdfs(): write (retrying)");
-                }
-            }
-            else
-                progress += err;
-        }
-
-        fFp->tell();
     }
-    else
+
+    if (errCount == MaxRetries)
+        throw ios_base::failure("OIDServer::writeData(): lseek failed "
+                                "too many times");
+
+    for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;)
     {
-        for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++)
+        err = fFp->write(&buf[progress], size - progress);
+
+        if (err < 0)
         {
-            seekerr = lseek(fFd, offset, SEEK_SET);
+            if (errno != EINTR)    // EINTR isn't really an error
-
-            if (seekerr < 0)
-                perror("OIDServer::writeData(): lseek");
-        }
-
-        if (errCount == MaxRetries)
-            throw ios_base::failure("OIDServer::writeData(): lseek failed "
-                                    "too many times");
-
-        for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;)
-        {
-            err = write(fFd, &buf[progress], size - progress);
-
-            if (err < 0)
             {
-                if (errno != EINTR)    // EINTR isn't really an error
+                errCount++;
-                {
+                perror("OIDServer::writeData(): write (retrying)");
-                    errCount++;
-                    perror("OIDServer::writeData(): write (retrying)");
-                }
             }
-            else
-                progress += err;
         }
+        else
+            progress += err;
     }
 
+    fFp->tell();
+
     if (errCount == MaxRetries)
         throw ios_base::failure("OIDServer::writeData(): write error");
 }
@@ -232,73 +170,37 @@ void OIDServer::readData(uint8_t* buf, off_t offset, int size) const
     if (size == 0)
         return;
 
-    // XXXPAT: Forcing the IDB* path.  Get rid of the fstream path when appropriate.
+    for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++)
-    if (true || IDBPolicy::useHdfs())
     {
-        for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++)
+        seekerr = fFp->seek(offset, SEEK_SET);
-        {
-            seekerr = fFp->seek(offset, SEEK_SET);
 
-            if (seekerr >= 0)
+        if (seekerr >= 0)
-                seekerr = fFp->tell(); // IDBDataFile may use fseek for seek.
+            seekerr = fFp->tell(); // IDBDataFile may use fseek for seek.
 
-            if (seekerr < 0)
+        if (seekerr < 0)
-                perror("OIDServer::readDataHdfs(): lseek");
+            perror("OIDServer::readData(): lseek");
-        }
-
-        if (errCount == MaxRetries)
-            throw ios_base::failure("OIDServer::readDataHdfs(): lseek failed "
-                                    "too many times");
-
-        for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;)
-        {
-            err = fFp->read(&buf[progress], size - progress);
-
-            if (err < 0)
-            {
-                if (errno != EINTR)    // EINTR isn't really an error
-                {
-                    errCount++;
-                    perror("OIDServer::readDataHdfs(): read (retrying)");
-                }
-            }
-            else if (err == 0)
-                throw EOFException();
-            else
-                progress += err;
-        }
     }
-    else
+
+    if (errCount == MaxRetries)
+        throw ios_base::failure("OIDServer::readData(): lseek failed "
+                                "too many times");
+
+    for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;)
     {
-        for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++)
+        err = fFp->read(&buf[progress], size - progress);
+
+        if (err < 0)
         {
-            seekerr = lseek(fFd, offset, SEEK_SET);
+            if (errno != EINTR)    // EINTR isn't really an error
-
-            if (seekerr < 0)
-                perror("OIDServer::readData(): lseek");
-        }
-
-        if (errCount == MaxRetries)
-            throw ios_base::failure("OIDServer::readData(): lseek failed "
-                                    "too many times");
-
-        for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;)
-        {
-            err = read(fFd, &buf[progress], size - progress);
-
-            if (err < 0)
             {
-                if (errno != EINTR)    // EINTR isn't really an error
+                errCount++;
-                {
+                perror("OIDServer::readData(): read (retrying)");
-                    errCount++;
-                    perror("OIDServer::readData(): read (retrying)");
-                }
             }
-            else if (err == 0)
-                throw EOFException();
-            else
-                progress += err;
         }
+        else if (err == 0)
+            throw EOFException();
+        else
+            progress += err;
     }
 
     if (errCount == MaxRetries)
@@ -349,14 +251,8 @@ void OIDServer::initializeBitmap() const
     }
 
     writeData(buf, 0, HeaderSize);
-
-    // reset buf to all 0's and write the bitmap
-    //for (i = 0; i < HeaderSize; i++)
-    //    buf[i] = 0;
-
-    //for (i = 0; i < bitmapSize; i += HeaderSize)
-    //    writeData(buf, HeaderSize + i, (bitmapSize - i > HeaderSize ? HeaderSize : bitmapSize - i));
     
+    // write the new bitmap file
     uint8_t *bitmapbuf = new uint8_t[bitmapSize];
     memset(bitmapbuf, 0, bitmapSize);
     writeData(bitmapbuf, HeaderSize, bitmapSize);
@@ -389,118 +285,55 @@ OIDServer::OIDServer() : fFp(NULL), fFd(-1)
         throw runtime_error(os.str());
     }
 
-    // XXXPAT: Forcing the IDB* path.
+    if (!IDBPolicy::exists(fFilename.c_str()))   //no bitmap file
-    if (true || IDBPolicy::useHdfs())
     {
-        if (!IDBPolicy::exists(fFilename.c_str()))   //no bitmap file
+        BRM::DBRM em;
+
+        if (!em.isEMEmpty())
         {
-            BRM::DBRM em;
+            os << "Extent Map not empty and " << fFilename << " not found. Setting system to read-only";
+            cerr << os.str() << endl;
+            log(os.str());
+            em.setReadOnly(true);
+            throw runtime_error(os.str());
+        }
 
-            if (!em.isEMEmpty())
+        fFp = IDBDataFile::open(IDBPolicy::getType(fFilename.c_str(), IDBPolicy::WRITEENG),
-            {
+                                fFilename.c_str(), "w+b", 0, 1);
-                os << "Extent Map not empty and " << fFilename << " not found. Setting system to read-only";
-                cerr << os.str() << endl;
-                log(os.str());
-                em.setReadOnly(true);
-                throw runtime_error(os.str());
-            }
 
-            fFp = IDBDataFile::open(IDBPolicy::getType(fFilename.c_str(), IDBPolicy::WRITEENG),
+        if (!fFp)
-                                    fFilename.c_str(), "w+b", 0, 1);
+        {
-
+            os << "Couldn't create oid bitmap file " << fFilename << ": " <<
-            if (!fFp)
+               strerror(errno);
-            {
+            log(os.str());
-                os << "Couldn't create oid bitmap file " << fFilename << ": " <<
+            throw ios_base::failure(os.str());
-                   strerror(errno);
+        }
-                log(os.str());
-                throw ios_base::failure(os.str());
-            }
 
 #ifndef _MSC_VER
-
+        if (fFp)
-            //FIXME:
+            chmod(fFilename.c_str(), 0664);   // XXXPAT: override umask at least for testing
-            //fchmod(fFd, 0666);   // XXXPAT: override umask at least for testing
-            if (fFp)
-                chmod(fFilename.c_str(), 0664);   // XXXPAT: override umask at least for testing
-
 #endif
 
-            try
+        try
-            {
-                initializeBitmap();
-            }
-            catch (...)
-            {
-                delete fFp;
-                fFp = NULL;
-                throw;
-            }
-        }
-        else
         {
-            fFp = IDBDataFile::open(IDBPolicy::getType(fFilename.c_str(), IDBPolicy::WRITEENG),
+            initializeBitmap();
-                                    fFilename.c_str(), "r+b", 0, 1);
+        }
-
+        catch (...)
-            if (!fFp)
+        {
-            {
+            delete fFp;
-                ostringstream os;
+            fFp = NULL;
-                os << "Couldn't open oid bitmap file" << fFilename << ": " <<
+            throw;
-                   strerror(errno);
-                log(os.str());
-                throw ios_base::failure(os.str());
-            }
         }
     }
     else
     {
-        if (access(fFilename.c_str(), F_OK) != 0) //no bitmap file
+        fFp = IDBDataFile::open(IDBPolicy::getType(fFilename.c_str(), IDBPolicy::WRITEENG),
+                                fFilename.c_str(), "r+b", 0, 1);
+
+        if (!fFp)
         {
-            BRM::DBRM em;
+            ostringstream os;
-
+            os << "Couldn't open oid bitmap file" << fFilename << ": " <<
-            if (!em.isEMEmpty())
-            {
-                os << "Extent Map not empty and " << fFilename << " not found. Setting system to read-only";
-                cerr << os.str() << endl;
-                log(os.str());
-                em.setReadOnly(true);
-                throw runtime_error(os.str());
-            }
-        }
-
-        fFd = open(fFilename.c_str(), O_CREAT | O_EXCL | O_RDWR | O_BINARY, 0664);
-
-        if (fFd >= 0)
-        {
-#ifndef _MSC_VER
-            //FIXME:
-            fchmod(fFd, 0666);   // XXXPAT: override umask at least for testing
-#endif
-
-            try
-            {
-                initializeBitmap();
-            }
-            catch (...)
-            {
-                close(fFd);
-                throw;
-            }
-        }
-        else if (errno == EEXIST)
-        {
-            fFd = open(fFilename.c_str(), O_RDWR | O_BINARY);
-
-            if (fFd < 0)
-            {
-                os << "Couldn't open oid bitmap file " << fFilename << ": " <<
-                   strerror(errno);
-                log(os.str());
-                throw ios_base::failure(os.str());
-            }
-        }
-        else
-        {
-            os << "Couldn't create oid bitmap file " << fFilename << ": " <<
                strerror(errno);
             log(os.str());
             throw ios_base::failure(os.str());
@@ -622,8 +455,7 @@ retry:
     {
         writeData(buf, offset, byteSize);
 
-        if (true || IDBPolicy::useHdfs())
+        fFp->flush();
-            fFp->flush();
 
         delete [] buf;
         return;
@@ -669,8 +501,7 @@ retry:
     {
         writeData(buf, offset, byteSize);
 
-        if (true || IDBPolicy::useHdfs())
+        fFp->flush();
-            fFp->flush();
 
         delete [] buf;
         return;
@@ -790,9 +621,7 @@ void OIDServer::patchFreelist(struct FEntry* freelist, int start, int num) const
     if (changed)
     {
         writeData(reinterpret_cast<uint8_t*>(freelist), 0, HeaderSize);
-
+        fFp->flush();
-        if (true || IDBPolicy::useHdfs())
-            fFp->flush();
     }
 }
 
@@ -819,8 +648,7 @@ int OIDServer::allocVBOID(uint16_t dbroot)
         throw;
     }
 
-    if (true || IDBPolicy::useHdfs())
+    fFp->flush();
-        fFp->flush();
 
     return ret;
 }
@@ -893,8 +721,7 @@ int OIDServer::allocOIDs(int num)
     writeData(reinterpret_cast<uint8_t*>(freelist), 0, HeaderSize);
     flipOIDBlock(bestMatchBegin, num, 0);
 
-    if (true || IDBPolicy::useHdfs())
+    fFp->flush();
-        fFp->flush();
 
     return bestMatchBegin;
 }

versioning/BRM/sessionmanagerserver.cpp

@@ -85,7 +85,7 @@ const uint32_t SessionManagerServer::SS_FORCE				= 1 << 5;   // In combination w
 const uint32_t SessionManagerServer::SS_QUERY_READY			= 1 << 6;   // Set by ProcManager when system is ready for queries
 
 
-SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0), txnidfd(-1)
+SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0)
 {
     config::Config* conf;
     string stmp;
@@ -119,26 +119,6 @@ SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0), txnidfd
 
     txnidFilename = conf->getConfig("SessionManager", "TxnIDFile");
 
-    if (false && !IDBPolicy::useHdfs())
-    {
-        txnidfd = open(txnidFilename.c_str(), O_RDWR | O_CREAT | O_BINARY, 0664);
-
-        if (txnidfd < 0)
-        {
-            perror("SessionManagerServer(): open");
-            throw runtime_error("SessionManagerServer: Could not open the transaction ID file");
-        }
-
-        //FIXME: do we need this on Win?
-#ifndef _MSC_VER
-        else
-        {
-            fchmod(txnidfd, 0664);
-        }
-
-#endif
-    }
-
     semValue = maxTxns;
     _verID = 0;
     _sysCatVerID = 0;
@@ -154,6 +134,9 @@ SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0), txnidfd
     }
 }
 
+SessionManagerServer::~SessionManagerServer()
+{
+}
 void SessionManagerServer::reset()
 {
     mutex.try_lock();
@@ -178,57 +161,7 @@ again:
     // If we fail to read a full four bytes for any value, then the
     // value isn't in the file, and we start with the default.
 
-    if (false && !IDBPolicy::useHdfs())
+    if (IDBPolicy::exists(txnidFilename.c_str()))
-    {
-        // Last transaction id
-        lseek(txnidfd, 0, SEEK_SET);
-        err = read(txnidfd, &lastTxnID, 4);
-
-        if (err < 0 && errno != EINTR)
-        {
-            perror("Sessionmanager::initSegment(): read");
-            throw runtime_error("SessionManagerServer: read failed, aborting");
-        }
-        else if (err < 0)
-            goto again;
-        else if (err == sizeof(int))
-            _verID = lastTxnID;
-
-        // last system catalog version id
-        err = read(txnidfd, &lastSysCatVerId, 4);
-
-        if (err < 0 && errno != EINTR)
-        {
-            perror("Sessionmanager::initSegment(): read");
-            throw runtime_error("SessionManagerServer: read failed, aborting");
-        }
-        else if (err < 0)
-            goto again;
-        else if (err == sizeof(int))
-            _sysCatVerID = lastSysCatVerId;
-
-        // System state. Contains flags regarding the suspend state of the system.
-        err = read(txnidfd, &systemState, 4);
-
-        if (err < 0 && errno == EINTR)
-        {
-            goto again;
-        }
-        else if (err == sizeof(int))
-        {
-            // Turn off the pending and force flags. They make no sense for a clean start.
-            // Turn off the ready flag. DMLProc will set it back on when
-            // initialized.
-            systemState &=
-                ~(SS_READY | SS_QUERY_READY | SS_SUSPEND_PENDING | SS_SHUTDOWN_PENDING | SS_ROLLBACK | SS_FORCE);
-        }
-        else
-        {
-            // else no problem. System state wasn't saved. Might be an upgraded system.
-            systemState = 0;
-        }
-    }
-    else if (IDBPolicy::exists(txnidFilename.c_str()))
     {
         scoped_ptr<IDBDataFile> txnidfp(IDBDataFile::open(
                                             IDBPolicy::getType(txnidFilename.c_str(),
@@ -297,26 +230,7 @@ again:
 */
 void SessionManagerServer::saveSystemState()
 {
-    if (false && !IDBPolicy::useHdfs())
-    {
-        int err = 0;
-        uint32_t lSystemState = systemState;
-
-        // We don't save the pending flags, the force flag or the ready flags.
-        lSystemState &= ~(SS_READY | SS_QUERY_READY | SS_SUSPEND_PENDING | SS_SHUTDOWN_PENDING | SS_FORCE);
-        lseek(txnidfd, 8, SEEK_SET);
-        err = write(txnidfd, &lSystemState, sizeof(int));
-
-        if (err < 0)
-        {
-            perror("SessionManagerServer::saveSystemState(): write(systemState)");
-            throw runtime_error("SessionManagerServer::saveSystemState(): write(systemState) failed");
-        }
-    }
-    else
-    {
         saveSMTxnIDAndState();
-    }
 }
 
 const QueryContext SessionManagerServer::verID()
@@ -377,25 +291,7 @@ const TxnID SessionManagerServer::newTxnID(const SID session, bool block, bool i
     if (isDDL)
         ++_sysCatVerID;
 
-    if (false && !IDBPolicy::useHdfs())
+    saveSMTxnIDAndState();
-    {
-        int filedata[2];
-        filedata[0] = _verID;
-        filedata[1] = _sysCatVerID;
-
-        lseek(txnidfd, 0, SEEK_SET);
-        int err = write(txnidfd, filedata, 8);
-
-        if (err < 0)
-        {
-            perror("SessionManagerServer::newTxnID(): write(verid)");
-            throw runtime_error("SessionManagerServer::newTxnID(): write(verid) failed");
-        }
-    }
-    else
-    {
-        saveSMTxnIDAndState();
-    }
 
     return ret;
 }

versioning/BRM/sessionmanagerserver.h

@@ -123,10 +123,7 @@ public:
      * It does not destroy the semaphores.  Those persist until the system
      * is shut down.
      */
-    virtual ~SessionManagerServer()
+    virtual ~SessionManagerServer();
-    {
-        if (txnidfd >= 0 ) close(txnidfd);
-    }
 
     /** @brief Gets the current version ID
      *
@@ -276,7 +273,6 @@ private:
 
     int maxTxns;  // the maximum number of concurrent transactions
     std::string txnidFilename;
-    int txnidfd;		// file descriptor for the "last txnid" file
     execplan::CalpontSystemCatalog::SCN _verID;
     execplan::CalpontSystemCatalog::SCN _sysCatVerID;
     uint32_t systemState;

versioning/BRM/slavecomm.cpp

@@ -65,7 +65,7 @@ namespace BRM
 {
 
 SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) :
-    slave(s), currentSaveFD(-1), currentSaveFile(NULL), journalh(NULL)
+    slave(s), currentSaveFile(NULL), journalh(NULL)
 #ifdef _MSC_VER
     , fPids(0), fMaxPids(64)
 #endif
@@ -138,21 +138,10 @@ SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) :
         firstSlave = true;
         journalName = savefile + "_journal";
         const char* filename = journalName.c_str();
-        uint32_t utmp = ::umask(0);
 
-        if (true || IDBPolicy::useHdfs())
+        journalh = IDBDataFile::open(
-        {
+                       IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "a", 0);
-            journalh = IDBDataFile::open(
+        if (journalh == NULL)
-                           IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "a", 0);
-        }
-        else
-        {
-            journal.open(filename, ios_base::binary | ios_base::out | ios_base::app);
-        }
-
-        ::umask(utmp);
-
-        if ((journal.is_open() == false) && (journalh == NULL))
             throw runtime_error("Could not open the BRM journal for writing!");
     }
     else
@@ -180,7 +169,7 @@ SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) :
 }
 
 SlaveComm::SlaveComm()
-    : currentSaveFD(-1), currentSaveFile(NULL), journalh(NULL)
+    : currentSaveFile(NULL), journalh(NULL)
 #ifdef _MSC_VER
     , fPids(0), fMaxPids(64)
 #endif
@@ -222,7 +211,6 @@ SlaveComm::~SlaveComm()
 
     if (firstSlave)
     {
-        close(currentSaveFD);
         delete currentSaveFile;
         currentSaveFile = NULL;
     }
@@ -2007,19 +1995,13 @@ void SlaveComm::do_confirm()
     if (firstSlave && (takeSnapshot ||
                        (journalCount >= snapshotInterval && snapshotInterval >= 0)))
     {
-        const char* filename = tmp.c_str();
+        if (!currentSaveFile)
-
-        if (false && !IDBPolicy::useHdfs() && currentSaveFD < 0)
-        {
-            currentSaveFD = open(filename, O_WRONLY | O_CREAT, 0664);
-        }
-        else if ((true || IDBPolicy::useHdfs()) && !currentSaveFile)
         {
             currentSaveFile = IDBDataFile::open(
-                                  IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "wb", 0);
+                                  IDBPolicy::getType(tmp.c_str(), IDBPolicy::WRITEENG), tmp.c_str(), "wb", 0);
         }
 
-        if (currentSaveFD < 0 && currentSaveFile == NULL)
+        if (currentSaveFile == NULL)
         {
             ostringstream os;
             os << "WorkerComm: failed to open the current savefile. errno: "
@@ -2028,9 +2010,7 @@ void SlaveComm::do_confirm()
             throw runtime_error(os.str());
         }
 
-#ifndef _MSC_VER
+        
-        chmod(filename, 0664);
-#endif
         tmp = savefile + (saveFileToggle ? 'A' : 'B');
         slave->saveState(tmp);
 #ifndef _MSC_VER
@@ -2038,74 +2018,33 @@ void SlaveComm::do_confirm()
 #endif
         int err = 0;
 
-        if (currentSaveFile)
+        // MCOL-1558.  Make the _current file relative to DBRMRoot.
+        string relative = tmp.substr(tmp.find_last_of('/') + 1);
+        err = currentSaveFile->write(relative.c_str(), relative.length());
+
+        if (err < (int) relative.length())
         {
-            // MCOL-1558.  Make the _current file relative to DBRMRoot.
+            ostringstream os;
-            string relative = tmp.substr(tmp.find_last_of('/') + 1);
+            os  << "WorkerComm: currentfile write() returned " << err
-            err = currentSaveFile->write(relative.c_str(), relative.length());
+                << " file pointer is " << currentSaveFile;
 
-            if (err < (int) relative.length())
+            if (err < 0)
-            {
+                os << " errno: " << strerror(errno);
-                ostringstream os;
-                os  << "WorkerComm: currentfile write() returned " << err
-                    << " file pointer is " << currentSaveFile;
 
-                if (err < 0)
+            log(os.str());
-                    os << " errno: " << strerror(errno);
-
-                log(os.str());
-            }
-
-            currentSaveFile->flush();
-            delete currentSaveFile;
-            currentSaveFile = NULL;
-            saveFileToggle = !saveFileToggle;
-
-            const char* filename = journalName.c_str();
-            //uint32_t utmp = ::umask(0);
-            delete journalh;
-            journalh = IDBDataFile::open(
-                           IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "w+b", 0);
-            //::umask(utmp);
-
-            if (!journalh)
-                throw runtime_error("Could not open the BRM journal for writing!");
         }
-        else
-        {
-            lseek(currentSaveFD, 0, SEEK_SET);
-            // MCOL-1558.  Make the _current file relative to DBRMRoot.
-            string relative = tmp.substr(tmp.find_last_of('/') + 1);
-            err = write(currentSaveFD, relative.c_str(), relative.length());
 
-            if (err < (int) relative.length())
+        currentSaveFile->flush();
-            {
+        delete currentSaveFile;
-                ostringstream os;
+        currentSaveFile = NULL;
-                os  << "WorkerComm: currentfile write() returned " << err
+        saveFileToggle = !saveFileToggle;
-                    << " fd is " << currentSaveFD;
 
-                if (err < 0)
+        delete journalh;
-                    os << " errno: " << strerror(errno);
+        journalh = IDBDataFile::open(
+                       IDBPolicy::getType(journalName.c_str(), IDBPolicy::WRITEENG), journalName.c_str(), "w+b", 0);
 
-                log(os.str());
+        if (!journalh)
-            }
+            throw runtime_error("Could not open the BRM journal for writing!");
-
-#ifdef _MSC_VER
-            //FIXME: Do we need to account for Windows EOL conversions?
-            _chsize_s(currentSaveFD, tmp.length());
-            _commit(currentSaveFD);
-#else
-            err = ftruncate(currentSaveFD, relative.length());
-            fsync(currentSaveFD);
-#endif
-            saveFileToggle = !saveFileToggle;
-
-            /* Is there a nicer way to truncate the file using an ofstream? */
-            journal.close();
-            uint32_t utmp = ::umask(0);
-            journal.open(journalName.c_str(), ios_base::binary | ios_base::out | ios_base::trunc);
-            ::umask(utmp);
-        }
 
         takeSnapshot = false;
         doSaveDelta = false;
@@ -2258,70 +2197,29 @@ int SlaveComm::replayJournal(string prefix)
 
     const char* filename = journalName.c_str();
 
-    if (true || IDBPolicy::useHdfs())
+    IDBDataFile* journalf = IDBDataFile::open(
+                                IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "rb", 0);
+
+    if (!journalf)
     {
-        IDBDataFile* journalf = IDBDataFile::open(
+        cout << "Error opening journal file " << fName << endl;
-                                    IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "rb", 0);
+        return -1;
-
-        if (!journalf)
-        {
-            cout << "Error opening journal file " << fName << endl;
-            return -1;
-        }
-
-        if (journalf->size() == 0)  // empty file, nothing to replay
-            return 0;
-
-        ssize_t readIn = 0;
-
-        do
-        {
-            readIn = journalf->read((char*) &len, sizeof(len));
-
-            if (readIn > 0)
-            {
-                cmd.needAtLeast(len);
-                readIn = journalf->read((char*) cmd.getInputPtr(), len);
-                cmd.advanceInputPtr(len);
-
-                try
-                {
-                    processCommand(cmd);
-                }
-                catch (exception& e)
-                {
-                    cout << e.what() << "  Journal replay was incomplete." << endl;
-                    slave->undoChanges();
-                    return -1;
-                }
-
-                slave->confirmChanges();
-                cmd.restart();
-                ret++;
-            }
-        }
-        while (readIn > 0);
     }
-    else
+
+    if (journalf->size() == 0)  // empty file, nothing to replay
+        return 0;
+
+    ssize_t readIn = 0;
+
+    do
     {
-        ifstream journalf;
+        readIn = journalf->read((char*) &len, sizeof(len));
-        journalf.open(filename, ios_base::in | ios_base::binary);
 
-        if (!journalf.is_open())
+        if (readIn > 0)
         {
-            cout << "Error opening journal file " << fName << endl;
-            return -1;
-        }
-
-        while (journalf)
-        {
-            journalf.read((char*) &len, sizeof(len));
-
-            if (!journalf)
-                break;
 
             cmd.needAtLeast(len);
-            journalf.read((char*) cmd.getInputPtr(), len);
+            readIn = journalf->read((char*) cmd.getInputPtr(), len);
             cmd.advanceInputPtr(len);
 
             try
@@ -2339,7 +2237,7 @@ int SlaveComm::replayJournal(string prefix)
             cmd.restart();
             ret++;
         }
-    }
+    } while (readIn > 0);
 
     return ret;
 }
@@ -2368,20 +2266,10 @@ void SlaveComm::saveDelta()
     {
         uint32_t len = delta.length();
 
-        if (true || IDBPolicy::useHdfs())
+        
-        {
+        journalh->write((const char*) &len, sizeof(len));
-            journalh->write((const char*) &len, sizeof(len));
+        journalh->write((const char*) delta.buf(), delta.length());
-            journalh->write((const char*) delta.buf(), delta.length());
+        journalh->flush();
-            journalh->flush();
-        }
-        else
-        {
-            journal.seekg(0, ios_base::end);
-            journal.write((const char*) &len, sizeof(len));
-            journal.write((const char*) delta.buf(), delta.length());
-            journal.sync();
-        }
-
         journalCount++;
     }
     catch (exception& e)

versioning/BRM/slavecomm.h

@@ -125,10 +125,8 @@ private:
     std::string savefile;
     bool release, die, firstSlave, saveFileToggle, takeSnapshot, doSaveDelta, standalone, printOnly;
     messageqcpp::ByteStream delta;
-    int currentSaveFD;
     idbdatafile::IDBDataFile* currentSaveFile;
     std::string journalName;
-    std::fstream journal;
     idbdatafile::IDBDataFile* journalh;
     int64_t snapshotInterval, journalCount;
     struct timespec MSG_TIMEOUT;

versioning/BRM/tablelockserver.cpp

@@ -63,41 +63,21 @@ void TableLockServer::save()
 
     const char* filename_p = filename.c_str();
 
-    if (true || IDBPolicy::useHdfs())
+    scoped_ptr<IDBDataFile> out(IDBDataFile::open(
+                                    IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
+                                    filename_p, "wb", 0));
+
+    if (!out)
+        throw runtime_error("TableLockServer::save():  could not open save file");
+
+    out->write((char*) &count, 4);
+
+    for (it = locks.begin(); it != locks.end(); ++it)
     {
-        scoped_ptr<IDBDataFile> out(IDBDataFile::open(
-                                        IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
-                                        filename_p, "wb", 0));
-
         if (!out)
-            throw runtime_error("TableLockServer::save():  could not open save file");
+            throw runtime_error("TableLockServer::save():  could not write save file");
 
-        out->write((char*) &count, 4);
+        it->second.serialize(out.get());
-
-        for (it = locks.begin(); it != locks.end(); ++it)
-        {
-            if (!out)
-                throw runtime_error("TableLockServer::save():  could not write save file");
-
-            it->second.serialize(out.get());
-        }
-    }
-    else
-    {
-        ofstream out(filename.c_str(), ios::trunc | ios::binary | ios::out );
-
-        if (!out)
-            throw runtime_error("TableLockServer::save():  could not open save file");
-
-        out.write((char*) &count, 4);
-
-        for (it = locks.begin(); it != locks.end(); ++it)
-        {
-            if (!out)
-                throw runtime_error("TableLockServer::save():  could not write save file");
-
-            it->second.serialize(out);
-        }
     }
 }
 
@@ -109,85 +89,42 @@ void TableLockServer::load()
     TableLockInfo tli;
 
     /* Need to standardize the file error handling */
-    if (true || IDBPolicy::useHdfs())
+    const char* filename_p = filename.c_str();
+    scoped_ptr<IDBDataFile>  in(IDBDataFile::open(
+                                    IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
+                                    filename_p, "rb", 0));
+
+    if (!in)
     {
-        const char* filename_p = filename.c_str();
+        ostringstream os;
-        scoped_ptr<IDBDataFile>  in(IDBDataFile::open(
+        os << "TableLockServer::load(): could not open the save file"
-                                        IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
+           << filename;
-                                        filename_p, "rb", 0));
+        log(os.str(), logging::LOG_TYPE_DEBUG);
+        return;
+    }
 
-        if (!in)
+    try
+    {
+        in->read((char*) &size, 4);
+
+        for (i = 0; i < size; i++)
         {
-            ostringstream os;
+            tli.deserialize(in.get());
-            os << "TableLockServer::load(): could not open the save file"
+            tli.id = sms->getUnique64();   // Need new #s...
-               << filename;
-            log(os.str(), logging::LOG_TYPE_DEBUG);
-            return;
-        }
 
-        try
+            if (tli.id == 0)	// 0 is an error code
-        {
+                tli.id = sms->getUnique64();
-            in->read((char*) &size, 4);
 
-            for (i = 0; i < size; i++)
+            locks[tli.id] = tli;
-            {
-                tli.deserialize(in.get());
-                tli.id = sms->getUnique64();   // Need new #s...
-
-                if (tli.id == 0)	// 0 is an error code
-                    tli.id = sms->getUnique64();
-
-                locks[tli.id] = tli;
-            }
-        }
-        catch (std::exception& e)
-        {
-            ostringstream os;
-            os << "TableLockServer::load(): could not load save file " << filename <<
-               " loaded " << i << "/" << size << " entries\n";
-            log(os.str(), logging::LOG_TYPE_DEBUG);
-            throw;
         }
     }
-    else
+    catch (std::exception& e)
     {
-        ifstream in(filename.c_str(), ios::binary | ios::in);
+        ostringstream os;
-
+        os << "TableLockServer::load(): could not load save file " << filename <<
-        if (!in)
+           " loaded " << i << "/" << size << " entries\n";
-        {
+        log(os.str(), logging::LOG_TYPE_DEBUG);
-            ostringstream os;
+        throw;
-            os << "TableLockServer::load(): could not open the save file"
-               << filename;
-            log(os.str(), logging::LOG_TYPE_DEBUG);
-            return;
-        }
-
-        in.exceptions(ios::failbit | ios::badbit);
-
-        try
-        {
-            in.read((char*) &size, 4);
-
-            for (i = 0; i < size; i++)
-            {
-                tli.deserialize(in);
-                tli.id = sms->getUnique64();   // Need new #s...
-
-                if (tli.id == 0)	// 0 is an error code
-                    tli.id = sms->getUnique64();
-
-                locks[tli.id] = tli;
-            }
-        }
-        catch (std::exception& e)
-        {
-            ostringstream os;
-
-            os << "TableLockServer::load(): could not load save file " << filename <<
-               " loaded " << i << "/" << size << " entries\n";
-            log(os.str(), logging::LOG_TYPE_DEBUG);
-            throw;
-        }
     }
 }
 

versioning/BRM/vbbm.cpp

@@ -21,7 +21,6 @@
  ****************************************************************************/
 
 #include <iostream>
-#include <fstream>
 #include <vector>
 #include <sstream>
 #include <sys/types.h>
@@ -980,54 +979,6 @@ void VBBM::setReadOnly()
  * 		struct VBBMEntry * numEntries
  */
 
-void VBBM::loadVersion1(IDBDataFile* in)
-{
-    int vbbmEntries, i;
-    VBBMEntry entry;
-
-    clear();
-
-    if (in->read((char*) &vbbmEntries, 4) != 4)
-    {
-        log_errno("VBBM::load()");
-        throw runtime_error("VBBM::load(): Failed to read entry number");
-    }
-
-    for (i = 0; i < vbbmEntries; i++)
-    {
-        if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry))
-        {
-            log_errno("VBBM::load()");
-            throw runtime_error("VBBM::load(): Failed to load entry");
-        }
-
-        insert(entry.lbid, entry.verID, entry.vbOID, entry.vbFBO, true);
-        //confirmChanges();
-        addVBFileIfNotExists(entry.vbOID);
-    }
-
-    /* This will load the saved file data from 2.2, but it is not compatible with
-     * 3.0+.  If enabled, take out the addVBFile..() call above
-     */
-#if 0
-    int dummy, nFiles;
-
-    in.read((char*) &nFiles, 4);
-    cout << "got nfiles = " << nFiles << endl;
-    in.read((char*) &dummy, 4);    // an unused var in 3.0+
-
-    while (vbbm->nFiles < nFiles)
-        growVBBM(true);  // this allocates one file, doesn't grow the main storage
-
-    in.read((char*) files, sizeof(VBFileMetadata) * nFiles);
-
-    for (i = 0; i < nFiles; i++)
-        cout << "file " << i << ": oid=" << files[i].OID << " size=" << files[i].fileSize
-             << " offset=" << files[i].nextOffset << endl;
-
-#endif
-
-}
 
 void VBBM::loadVersion2(IDBDataFile* in)
 {
@@ -1091,19 +1042,6 @@ void VBBM::loadVersion2(IDBDataFile* in)
     for (i = 0; i < vbbmEntries; i++)
         insert(loadedEntries[i].lbid, loadedEntries[i].verID, loadedEntries[i].vbOID,
           loadedEntries[i].vbFBO, true);
-    
-    /*
-    for (i = 0; i < vbbmEntries; i++)
-    {
-        if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry))
-        {
-            log_errno("VBBM::load()");
-            throw runtime_error("VBBM::load(): Failed to load entry");
-        }
-
-        insert(entry.lbid, entry.verID, entry.vbOID, entry.vbFBO, true);
-    }
-    */
 
 }
 
@@ -1138,9 +1076,6 @@ void VBBM::load(string filename)
 
     switch (magic)
     {
-        case VBBM_MAGIC_V1:
-            loadVersion1(in.get());
-            break;
 
         case VBBM_MAGIC_V2:
             loadVersion2(in.get());
@@ -1161,59 +1096,38 @@ void VBBM::save(string filename)
     int i;
     int var;
 
-    // XXXPAT: forcing the IDB* path.  Delete the fstream path when appropriate.
+    const char* filename_p = filename.c_str();
-    if (true || IDBPolicy::useHdfs())
+    scoped_ptr<IDBDataFile> out(IDBDataFile::open(
+                                    IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
+                                    filename_p, "wb", IDBDataFile::USE_VBUF));
+
+    if (!out)
     {
-        const char* filename_p = filename.c_str();
+        log_errno("VBBM::save()");
-        scoped_ptr<IDBDataFile> out(IDBDataFile::open(
+        throw runtime_error("VBBM::save(): Failed to open the file");
-                                        IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
+    }
-                                        filename_p, "wb", IDBDataFile::USE_VBUF));
 
-        if (!out)
+    var = VBBM_MAGIC_V2;
+    int bytesWritten = 0;
+    int bytesToWrite = 12;
+    bytesWritten += out->write((char*) &var, 4);
+    bytesWritten += out->write((char*) &vbbm->vbCurrentSize, 4);
+    bytesWritten += out->write((char*) &vbbm->nFiles, 4);
+
+    bytesWritten += out->write((char*) files, sizeof(VBFileMetadata) * vbbm->nFiles);
+    bytesToWrite += sizeof(VBFileMetadata) * vbbm->nFiles;
+
+    int first = -1, last = -1, err;
+    size_t progress, writeSize;
+    
+    for (i = 0; i < vbbm->vbCapacity; i++)
+    {
+        if (storage[i].lbid != -1 && first == -1)
+            first = i;
+        else if (storage[i].lbid == -1 && first != -1)
         {
-            log_errno("VBBM::save()");
+            last = i;
-            throw runtime_error("VBBM::save(): Failed to open the file");
+            writeSize = (last - first) * sizeof(VBBMEntry);
-        }
-
-        var = VBBM_MAGIC_V2;
-        int bytesWritten = 0;
-        int bytesToWrite = 12;
-        bytesWritten += out->write((char*) &var, 4);
-        bytesWritten += out->write((char*) &vbbm->vbCurrentSize, 4);
-        bytesWritten += out->write((char*) &vbbm->nFiles, 4);
-
-        bytesWritten += out->write((char*) files, sizeof(VBFileMetadata) * vbbm->nFiles);
-        bytesToWrite += sizeof(VBFileMetadata) * vbbm->nFiles;
-
-        int first = -1, last = -1, err;
-        size_t progress, writeSize;
-        
-        for (i = 0; i < vbbm->vbCapacity; i++)
-        {
-            if (storage[i].lbid != -1 && first == -1)
-                first = i;
-            else if (storage[i].lbid == -1 && first != -1)
-            {
-                last = i;
-                writeSize = (last - first) * sizeof(VBBMEntry);
-                progress = 0;
-                char *writePos = (char *) &storage[first];
-                while (progress < writeSize)
-                {
-                    err = out->write(writePos + progress, writeSize - progress);
-                    if (err < 0)
-                    {
-                        log_errno("VBBM::save()");
-                        throw runtime_error("VBBM::save(): Failed to write the file");
-                    }
-                    progress += err;
-                }
-                first = -1;
-            }
-        }
-        if (first != -1)
-        {
-            writeSize = (vbbm->vbCapacity - first) * sizeof(VBBMEntry);
             progress = 0;
             char *writePos = (char *) &storage[first];
             while (progress < writeSize)
@@ -1226,49 +1140,27 @@ void VBBM::save(string filename)
                 }
                 progress += err;
             }
+            first = -1;
         }
 
-        /*
-        for (i = 0; i < vbbm->vbCapacity; i++)
-        {
-            if (storage[i].lbid != -1)
-            {
-                bytesToWrite += sizeof(VBBMEntry);
-                bytesWritten += out->write((char*)&storage[i], sizeof(VBBMEntry));
-            }
-        }
-
-        if (bytesWritten != bytesToWrite)
-        {
-            log_errno("VBBM::save()");
-            throw runtime_error("VBBM::save(): Failed to write the file");
-        }
-        */
     }
-    else
+    if (first != -1)
     {
-        ofstream out;
+        writeSize = (vbbm->vbCapacity - first) * sizeof(VBBMEntry);
-        out.open(filename.c_str(), ios_base::trunc | ios_base::out | ios_base::binary);
+        progress = 0;
-
+        char *writePos = (char *) &storage[first];
-        if (!out)
+        while (progress < writeSize)
         {
-            log_errno("VBBM::save()");
+            err = out->write(writePos + progress, writeSize - progress);
-            throw runtime_error("VBBM::save(): Failed to open the file");
+            if (err < 0)
+            {
+                log_errno("VBBM::save()");
+                throw runtime_error("VBBM::save(): Failed to write the file");
+            }
+            progress += err;
         }
-
-        out.exceptions(ios_base::badbit);
-
-        var = VBBM_MAGIC_V2;
-        out.write((char*) &var, 4);
-        out.write((char*) &vbbm->vbCurrentSize, 4);
-        out.write((char*) &vbbm->nFiles, 4);
-
-        out.write((char*) files, sizeof(VBFileMetadata) * vbbm->nFiles);
-
-        for (i = 0; i < vbbm->vbCapacity; i++)
-            if (storage[i].lbid != -1)
-                out.write((char*)&storage[i], sizeof(VBBMEntry));
     }
+    
 
 #if 0
     cout << "saving... nfiles=" << vbbm->nFiles << "\n";

versioning/BRM/vbbm.h

@@ -214,7 +214,6 @@ public:
 
     EXPORT void clear();
     EXPORT void load(std::string filename);
-    EXPORT void loadVersion1(idbdatafile::IDBDataFile* in);
     EXPORT void loadVersion2(idbdatafile::IDBDataFile* in);
     EXPORT void save(std::string filename);
 

versioning/BRM/vss.cpp

@@ -21,7 +21,6 @@
  ****************************************************************************/
 
 #include <iostream>
-#include <fstream>
 #include <sstream>
 //#define NDEBUG
 #include <cassert>
@@ -1294,57 +1293,36 @@ void VSS::save(string filename)
     int i;
     struct Header header;
 
-    // XXXPAT: Forcing the IDB* path to run.  Delete the fstream path when appropriate.
+    const char* filename_p = filename.c_str();
-    if (true || IDBPolicy::useHdfs())
+    scoped_ptr<IDBDataFile> out(IDBDataFile::open(
+                                    IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
+                                    filename_p, "wb", IDBDataFile::USE_VBUF));
+
+    if (!out)
     {
-        const char* filename_p = filename.c_str();
+        log_errno("VSS::save()");
-        scoped_ptr<IDBDataFile> out(IDBDataFile::open(
+        throw runtime_error("VSS::save(): Failed to open the file");
-                                        IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
+    }
-                                        filename_p, "wb", IDBDataFile::USE_VBUF));
 
-        if (!out)
+    header.magic = VSS_MAGIC_V1;
-        {
+    header.entries = vss->currentSize;
-            log_errno("VSS::save()");
-            throw runtime_error("VSS::save(): Failed to open the file");
-        }
 
-        header.magic = VSS_MAGIC_V1;
+    if (out->write((char*)&header, sizeof(header)) != sizeof(header))
-        header.entries = vss->currentSize;
+    {
+        log_errno("VSS::save()");
+        throw runtime_error("VSS::save(): Failed to write header to the file");
+    }
 
-        if (out->write((char*)&header, sizeof(header)) != sizeof(header))
+    int first = -1, last = -1, err;
+    size_t progress, writeSize;
+    for (i = 0; i < vss->capacity; i++)
+    {
+        if (storage[i].lbid != -1 && first == -1)
+            first = i;
+        else if (storage[i].lbid == -1 && first != -1)
         {
-            log_errno("VSS::save()");
+            last = i;
-            throw runtime_error("VSS::save(): Failed to write header to the file");
+            writeSize = (last - first) * sizeof(VSSEntry);
-        }
-
-        int first = -1, last = -1, err;
-        size_t progress, writeSize;
-        for (i = 0; i < vss->capacity; i++)
-        {
-            if (storage[i].lbid != -1 && first == -1)
-                first = i;
-            else if (storage[i].lbid == -1 && first != -1)
-            {
-                last = i;
-                writeSize = (last - first) * sizeof(VSSEntry);
-                progress = 0;
-                char *writePos = (char *) &storage[first];
-                while (progress < writeSize)
-                {
-                    err = out->write(writePos + progress, writeSize - progress);
-                    if (err < 0)
-                    {
-                        log_errno("VSS::save()");
-                        throw runtime_error("VSS::save(): Failed to write the file");
-                    }
-                    progress += err;
-                }
-                first = -1;
-            }
-        }
-        if (first != -1)
-        {
-            writeSize = (vss->capacity - first) * sizeof(VSSEntry);
             progress = 0;
             char *writePos = (char *) &storage[first];
             while (progress < writeSize)
@@ -1357,54 +1335,24 @@ void VSS::save(string filename)
                 }
                 progress += err;
             }
+            first = -1;
         }
-        
-        /*
-        for (i = 0; i < vss->capacity; i++)
-        {
-            if (storage[i].lbid != -1)
-            {
-                if (out->write((char*)&storage[i], sizeof(VSSEntry)) != sizeof(VSSEntry))
-                {
-                    log_errno("VSS::save()");
-                    throw runtime_error("VSS::save(): Failed to write vss entry to the file");
-                }
-            }
-        }
-        */
     }
-    else
+    if (first != -1)
     {
-        ofstream out;
+        writeSize = (vss->capacity - first) * sizeof(VSSEntry);
-        out.open(filename.c_str(), ios_base::trunc | ios_base::out | ios_base::binary);
+        progress = 0;
-        //::umask(utmp);
+        char *writePos = (char *) &storage[first];
-
+        while (progress < writeSize)
-        if (!out)
         {
-            log_errno("VSS::save()");
+            err = out->write(writePos + progress, writeSize - progress);
-            throw runtime_error("VSS::save(): Failed to open the file");
+            if (err < 0)
+            {
+                log_errno("VSS::save()");
+                throw runtime_error("VSS::save(): Failed to write the file");
+            }
+            progress += err;
         }
-
-        out.exceptions(ios_base::badbit);
-
-        header.magic = VSS_MAGIC_V1;
-        header.entries = vss->currentSize;
-
-        try
-        {
-            out.write((char*)&header, sizeof(header));
-
-            for (i = 0; i < vss->capacity; i++)
-                if (storage[i].lbid != -1)
-                    out.write((char*)&storage[i], sizeof(VSSEntry));
-        }
-        catch (std::exception& e)
-        {
-            out.close();
-            throw;
-        }
-
-        out.close();
     }
 
 }
@@ -1517,94 +1465,11 @@ void VSS::load(string filename)
     for (i = 0; i < header.entries; i++)
         insert(loadedEntries[i].lbid, loadedEntries[i].verID, loadedEntries[i].vbFlag, 
           loadedEntries[i].locked, true);
-
-    /*
-    for (i = 0; i < header.entries; i++)
-    {
-        if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry))
-        {
-            log_errno("VSS::load()");
-            throw runtime_error("VSS::load(): Failed to read entry");
-        }
-
-        insert(entry.lbid, entry.verID, entry.vbFlag, entry.locked, true);
-    }
-
-    */
     
     //time2 = microsec_clock::local_time();
     //cout << "done loading " << time2 << " duration: " << time2-time1 << endl;
 }
 
-#ifndef __LP64__
-//This code is OBE now that the structs are padded correctly
-struct VSSEntry_
-{
-    LBID_t lbid;
-    VER_t verID;
-    bool vbFlag;
-    bool locked;
-    int next;
-    uint32_t pad1;
-};
-
-void VSS::load64(string filename)
-{
-    int i;
-    struct Header header;
-    struct VSSEntry_ entry;
-
-    const char* filename_p = filename.c_str();
-    scoped_ptr<IDBDataFile>  in(IDBDataFile::open(
-                                    IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
-                                    filename_p, "rb", 0));
-
-    if (!in)
-    {
-        log_errno("VSS::load()");
-        throw runtime_error("VSS::load(): Failed to open the file");
-    }
-
-    if (in->read((char*)&header, sizeof(header)) != sizeof(header))
-    {
-        log_errno("VSS::load()");
-        throw runtime_error("VSS::load(): Failed to read header");
-    }
-
-    if (header.magic != VSS_MAGIC_V1)
-    {
-        log("VSS::load(): Bad magic.  Not a VSS file?");
-        throw runtime_error("VSS::load(): Bad magic.  Not a VSS file?");
-    }
-
-    if (header.entries < 0)
-    {
-        log("VSS::load(): Bad size.  Not a VSS file?");
-        throw runtime_error("VSS::load(): Bad size.  Not a VSS file?");
-    }
-
-    for (i = 0; i < vss->capacity; i++)
-        storage[i].lbid = -1;
-
-    for (i = 0; i < vss->numHashBuckets; i++)
-        hashBuckets[i] = -1;
-
-    vss->currentSize = 0;
-    vss->lockedEntryCount = 0;
-    vss->LWM = 0;
-
-    for (i = 0; i < header.entries; i++)
-    {
-        if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry))
-        {
-            log_errno("VSS::load()");
-            throw runtime_error("VSS::load(): Failed to read entry");
-        }
-
-        insert(entry.lbid, entry.verID, entry.vbFlag, entry.locked, true);
-    }
-}
-#endif
 
 #ifdef BRM_DEBUG
 // read lock

versioning/BRM/vss.h

@@ -250,10 +250,6 @@ public:
 
     EXPORT void clear();
     EXPORT void load(std::string filename);
-#ifndef __LP64__
-    //This method is OBE now that the structs are padded correctly
-    EXPORT void load64(std::string filename);
-#endif
     EXPORT void save(std::string filename);
 
 #ifdef BRM_DEBUG