database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-27 21:01:50 +03:00
Code Activity

Merge branch 'develop' into S3-project

Browse Source
This commit is contained in:
Patrick LeBlanc
2019-08-05 16:34:45 -05:00
parent 99f8a4feb7 d30745d37c
commit 295c638c18
15 changed files with 356 additions and 1279 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
oamapps/postConfigure/installer.cpp
View File

@ -989,7 +989,7 @@ int main(int argc, char* argv[])
system(cmd.c_str());
if (oam.checkLogStatus(logFile, "System Catalog created") )
cout << endl << "System Catalog Successfull Created" << endl;
cout << endl << "System Catalog Successfully Created" << endl;
else
{
if ( oam.checkLogStatus(logFile, "System catalog appears to exist") )

1
oamapps/postConfigure/postConfigure.cpp
View File

@ -808,6 +808,7 @@ int main(int argc, char* argv[])
try
{
sysConfig->setConfig(InstallSection, "MySQLPort", mysqlPort);
sysConfig->setConfig("CrossEngineSupport", "Port", mysqlPort);
}
catch (...)
{}

4
oamapps/postConfigure/quick_installer_single_server.sh
View File

@ -6,13 +6,13 @@
for arg in "$@"; do
if [ `expr -- "$arg" : '--help'` -eq 6 ]; then
echo "Usage ./quick_installer_multi_server.sh"
echo "Usage ./quick_installer_single_server.sh"
echo ""
echo "Quick Installer for a Single Server MariaDB ColumnStore Install"
echo ""
exit 1
else
echo "quick_installer_multi_server.sh: ignoring unknown argument: $arg" 1>&2
echo "quick_installer_single_server.sh: ignoring unknown argument: $arg" 1>&2
fi
done

11
versioning/BRM/blockresolutionmanager.cpp
View File

@ -109,20 +109,9 @@ int BlockResolutionManager::saveState(string filename) throw()
// truncate teh file if already exists since no truncate in HDFS.
const char* filename = journalFilename.c_str();
if (true || IDBPolicy::useHdfs())
{
IDBDataFile* journal = IDBDataFile::open(
IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "wb", 0);
delete journal;
}
else
{
ofstream journal;
uint32_t utmp = ::umask(0);
journal.open(filename, ios_base::out | ios_base::trunc | ios_base::binary);
journal.close();
::umask(utmp);
}
vbbm.save(vbbmFilename);
vss.save(vssFilename);

211
versioning/BRM/extentmap.cpp
View File

@ -1133,84 +1133,6 @@ void ExtentMap::reserveLBIDRange(LBID_t start, uint8_t size)
... (* numFL)
*/
void ExtentMap::loadVersion4(ifstream& in)
{
int emNumElements, flNumElements;
in.read((char*) &emNumElements, sizeof(int));
in.read((char*) &flNumElements, sizeof(int));
idbassert(emNumElements > 0);
void *fExtentMapPtr = static_cast<void*>(fExtentMap);
memset(fExtentMapPtr, 0, fEMShminfo->allocdSize);
fEMShminfo->currentSize = 0;
// init the free list
memset(fFreeList, 0, fFLShminfo->allocdSize);
fFreeList[0].size = (1 << 26); // 2^36 LBIDs
fFLShminfo->currentSize = sizeof(InlineLBIDRange);
// @Bug 3498
// Calculate how big an extent map we're going to need and allocate it in one call
if ((fEMShminfo->allocdSize / sizeof(EMEntry)) < (unsigned)emNumElements)
{
size_t nrows = emNumElements;
//Round up to the nearest EM_INCREMENT_ROWS
if ((nrows % EM_INCREMENT_ROWS) != 0)
{
nrows /= EM_INCREMENT_ROWS;
nrows++;
nrows *= EM_INCREMENT_ROWS;
}
growEMShmseg(nrows);
}
for (int i = 0; i < emNumElements; i++)
{
in.read((char*) &fExtentMap[i], sizeof(EMEntry));
reserveLBIDRange(fExtentMap[i].range.start, fExtentMap[i].range.size);
//@bug 1911 - verify status value is valid
if (fExtentMap[i].status < EXTENTSTATUSMIN ||
fExtentMap[i].status > EXTENTSTATUSMAX)
fExtentMap[i].status = EXTENTAVAILABLE;
}
fEMShminfo->currentSize = emNumElements * sizeof(EMEntry);
#ifdef DUMP_EXTENT_MAP
EMEntry* emSrc = fExtentMap;
cout << "lbid\tsz\toid\tfbo\thwm\tpart#\tseg#\tDBRoot\twid\tst\thi\tlo\tsq\tv" << endl;
for (int i = 0; i < emNumElements; i++)
{
cout <<
emSrc[i].start
<< '\t' << emSrc[i].size
<< '\t' << emSrc[i].fileID
<< '\t' << emSrc[i].blockOffset
<< '\t' << emSrc[i].HWM
<< '\t' << emSrc[i].partitionNum
<< '\t' << emSrc[i].segmentNum
<< '\t' << emSrc[i].dbRoot
<< '\t' << emSrc[i].status
<< '\t' << emSrc[i].partition.cprange.hi_val
<< '\t' << emSrc[i].partition.cprange.lo_val
<< '\t' << emSrc[i].partition.cprange.sequenceNum
<< '\t' << (int)(emSrc[i].partition.cprange.isValid)
<< endl;
}
cout << "Free list entries:" << endl;
cout << "start\tsize" << endl;
for (int i = 0; i < flNumElements; i++)
cout << fFreeList[i].start << '\t' << fFreeList[i].size << endl;
#endif
}
void ExtentMap::loadVersion4(IDBDataFile* in)
{
@ -1264,7 +1186,6 @@ void ExtentMap::loadVersion4(IDBDataFile* in)
log_errno("ExtentMap::loadVersion4(): read ");
throw runtime_error("ExtentMap::loadVersion4(): read failed. Check the error log.");
}
progress += (uint) err;
}
@ -1337,9 +1258,6 @@ void ExtentMap::load(const string& filename, bool fixFL)
throw;
}
// XXXPAT: Forcing the IDB path. Remove the fstream path once we see this works.
if (true || IDBPolicy::useHdfs())
{
const char* filename_p = filename.c_str();
scoped_ptr<IDBDataFile> in(IDBDataFile::open(
IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
@ -1366,51 +1284,12 @@ void ExtentMap::load(const string& filename, bool fixFL)
throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image");
}
}
catch (...)
else // fstream path to be remove
{
releaseFreeList(WRITE);
releaseEMEntryTable(WRITE);
throw;
}
}
else
{
ifstream in;
in.open(filename.c_str(), ios_base::in | ios_base::binary);
if (!in)
{
log_errno("ExtentMap::load(): open");
releaseFreeList(WRITE);
releaseEMEntryTable(WRITE);
throw ios_base::failure("ExtentMap::load(): open failed. Check the error log.");
}
in.exceptions(ios_base::badbit | ios_base::failbit);
try
{
int emVersion;
in.read((char*) &emVersion, sizeof(int));
if (emVersion == EM_MAGIC_V4)
loadVersion4(in);
else
{
log("ExtentMap::load(): That file is not a valid ExtentMap image");
throw runtime_error("ExtentMap::load(): That file is not a valid ExtentMap image");
}
}
catch (...)
{
in.close();
releaseFreeList(WRITE);
releaseEMEntryTable(WRITE);
throw;
}
in.close();
}
releaseFreeList(WRITE);
releaseEMEntryTable(WRITE);
@ -1453,11 +1332,6 @@ void ExtentMap::save(const string& filename)
throw runtime_error("ExtentMap::save(): got request to save an empty BRM");
}
// XXXPAT: I don't know why there are two options here. It can just use the IDBDataFile stuff.
// Forcing the IDB option to execute for now. Leaving the old fstream version there in case we find there's
// a case the IDB option doesn't work.
if (true || IDBPolicy::useHdfs())
{
const char* filename_p = filename.c_str();
scoped_ptr<IDBDataFile> out(IDBDataFile::open(
IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
@ -1520,6 +1394,7 @@ void ExtentMap::save(const string& filename)
}
first = -1;
}
}
if (first != -1)
{
@ -1557,88 +1432,6 @@ void ExtentMap::save(const string& filename)
progress += err;
}
}
else // this is the fstream version to be expired
{
ofstream out;
// Make em writes to disk use a buffer size of StrmBufSize bytes (instead of the default 8K)
const unsigned StrmBufSize = 1 * 1024 * 1024;
scoped_array<char> buf(new char[StrmBufSize]);
out.rdbuf()->pubsetbuf(buf.get(), StrmBufSize);
utmp = ::umask(0);
out.open(filename.c_str(), ios_base::out | ios_base::binary);
::umask(utmp);
if (!out)
{
log_errno("ExtentMap::save(): open");
releaseFreeList(READ);
releaseEMEntryTable(READ);
throw ios_base::failure("ExtentMap::save(): open failed. Check the error log.");
}
out.exceptions(ios_base::badbit);
loadSize[0] = EM_MAGIC_V4;
loadSize[1] = fEMShminfo->currentSize / sizeof(EMEntry);
loadSize[2] = fFLShminfo->allocdSize / sizeof(InlineLBIDRange); // needs to send all entries
try
{
out.write((char*)loadSize, 3 * sizeof(int));
}
catch (...)
{
out.close();
releaseFreeList(READ);
releaseEMEntryTable(READ);
throw;
}
allocdSize = fEMShminfo->allocdSize / sizeof(EMEntry);
for (i = 0; i < allocdSize; i++)
{
if (fExtentMap[i].range.size > 0)
{
try
{
out.write((char*) &fExtentMap[i], sizeof(EMEntry));
}
catch (...)
{
out.close();
releaseFreeList(READ);
releaseEMEntryTable(READ);
throw;
}
}
}
allocdSize = fFLShminfo->allocdSize / sizeof(InlineLBIDRange);
for (i = 0; i < allocdSize; i++)
{
// if (fFreeList[i].size > 0) {
try
{
out.write((char*) &fFreeList[i], sizeof(InlineLBIDRange));
}
catch (...)
{
out.close();
releaseFreeList(READ);
releaseEMEntryTable(READ);
throw;
}
// }
}
out.close();
}
releaseFreeList(READ);
releaseEMEntryTable(READ);

183
versioning/BRM/oidserver.cpp
View File

@ -117,34 +117,6 @@ namespace BRM
boost::mutex OIDServer::fMutex;
#if 0
void OIDServer::lockFile() const
{
int err, errCount;
int errnoSave = 0;
for (errCount = 0, err = -1; err != 0 && errCount < MaxRetries;)
{
err = flock(fFd, LOCK_EX);
if (err < 0 && errno != EINTR) // EINTR isn't really an error
{
errCount++;
errnoSave = errno; // save errno because perror may overwrite
perror("OIDServer::lockFile(): flock (retrying)");
}
}
if (errCount == MaxRetries)
{
ostringstream oss;
oss << "OIDServer::lockFile(): flock error: " << strerror(errnoSave);
throw ios_base::failure(oss.str());
}
}
#endif
void OIDServer::writeData(uint8_t* buf, off_t offset, int size) const
{
int errCount, err, progress;
@ -153,9 +125,6 @@ void OIDServer::writeData(uint8_t* buf, off_t offset, int size) const
if (size == 0)
return;
// XXXPAT: Forcing the IDB* path. Get rid of the fstream path when appropriate.
if (true || IDBPolicy::useHdfs())
{
for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++)
{
seekerr = fFp->seek(offset, SEEK_SET);
@ -163,38 +132,6 @@ void OIDServer::writeData(uint8_t* buf, off_t offset, int size) const
if (seekerr >= 0)
seekerr = fFp->tell(); // IDBDataFile may use fseek for seek.
if (seekerr < 0)
perror("OIDServer::writeDataHdfs(): lseek");
}
if (errCount == MaxRetries)
throw ios_base::failure("OIDServer::writeDataHdfs(): lseek failed "
"too many times");
for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;)
{
err = fFp->write(&buf[progress], size - progress);
if (err < 0)
{
if (errno != EINTR) // EINTR isn't really an error
{
errCount++;
perror("OIDServer::writeDataHdfs(): write (retrying)");
}
}
else
progress += err;
}
fFp->tell();
}
else
{
for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++)
{
seekerr = lseek(fFd, offset, SEEK_SET);
if (seekerr < 0)
perror("OIDServer::writeData(): lseek");
}
@ -205,7 +142,7 @@ void OIDServer::writeData(uint8_t* buf, off_t offset, int size) const
for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;)
{
err = write(fFd, &buf[progress], size - progress);
err = fFp->write(&buf[progress], size - progress);
if (err < 0)
{
@ -218,7 +155,8 @@ void OIDServer::writeData(uint8_t* buf, off_t offset, int size) const
else
progress += err;
}
}
fFp->tell();
if (errCount == MaxRetries)
throw ios_base::failure("OIDServer::writeData(): write error");
@ -232,9 +170,6 @@ void OIDServer::readData(uint8_t* buf, off_t offset, int size) const
if (size == 0)
return;
// XXXPAT: Forcing the IDB* path. Get rid of the fstream path when appropriate.
if (true || IDBPolicy::useHdfs())
{
for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++)
{
seekerr = fFp->seek(offset, SEEK_SET);
@ -242,38 +177,6 @@ void OIDServer::readData(uint8_t* buf, off_t offset, int size) const
if (seekerr >= 0)
seekerr = fFp->tell(); // IDBDataFile may use fseek for seek.
if (seekerr < 0)
perror("OIDServer::readDataHdfs(): lseek");
}
if (errCount == MaxRetries)
throw ios_base::failure("OIDServer::readDataHdfs(): lseek failed "
"too many times");
for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;)
{
err = fFp->read(&buf[progress], size - progress);
if (err < 0)
{
if (errno != EINTR) // EINTR isn't really an error
{
errCount++;
perror("OIDServer::readDataHdfs(): read (retrying)");
}
}
else if (err == 0)
throw EOFException();
else
progress += err;
}
}
else
{
for (errCount = 0; errCount < MaxRetries && seekerr != offset; errCount++)
{
seekerr = lseek(fFd, offset, SEEK_SET);
if (seekerr < 0)
perror("OIDServer::readData(): lseek");
}
@ -284,7 +187,7 @@ void OIDServer::readData(uint8_t* buf, off_t offset, int size) const
for (progress = 0, errCount = 0; progress < size && errCount < MaxRetries;)
{
err = read(fFd, &buf[progress], size - progress);
err = fFp->read(&buf[progress], size - progress);
if (err < 0)
{
@ -299,7 +202,6 @@ void OIDServer::readData(uint8_t* buf, off_t offset, int size) const
else
progress += err;
}
}
if (errCount == MaxRetries)
throw ios_base::failure("OIDServer::readData(): read error");
@ -350,13 +252,7 @@ void OIDServer::initializeBitmap() const
writeData(buf, 0, HeaderSize);
// reset buf to all 0's and write the bitmap
//for (i = 0; i < HeaderSize; i++)
// buf[i] = 0;
//for (i = 0; i < bitmapSize; i += HeaderSize)
// writeData(buf, HeaderSize + i, (bitmapSize - i > HeaderSize ? HeaderSize : bitmapSize - i));
// write the new bitmap file
uint8_t *bitmapbuf = new uint8_t[bitmapSize];
memset(bitmapbuf, 0, bitmapSize);
writeData(bitmapbuf, HeaderSize, bitmapSize);
@ -389,9 +285,6 @@ OIDServer::OIDServer() : fFp(NULL), fFd(-1)
throw runtime_error(os.str());
}
// XXXPAT: Forcing the IDB* path.
if (true || IDBPolicy::useHdfs())
{
if (!IDBPolicy::exists(fFilename.c_str())) //no bitmap file
{
BRM::DBRM em;
@ -417,12 +310,8 @@ OIDServer::OIDServer() : fFp(NULL), fFd(-1)
}
#ifndef _MSC_VER
//FIXME:
//fchmod(fFd, 0666); // XXXPAT: override umask at least for testing
if (fFp)
chmod(fFilename.c_str(), 0664); // XXXPAT: override umask at least for testing
#endif
try
@ -450,62 +339,6 @@ OIDServer::OIDServer() : fFp(NULL), fFd(-1)
throw ios_base::failure(os.str());
}
}
}
else
{
if (access(fFilename.c_str(), F_OK) != 0) //no bitmap file
{
BRM::DBRM em;
if (!em.isEMEmpty())
{
os << "Extent Map not empty and " << fFilename << " not found. Setting system to read-only";
cerr << os.str() << endl;
log(os.str());
em.setReadOnly(true);
throw runtime_error(os.str());
}
}
fFd = open(fFilename.c_str(), O_CREAT | O_EXCL | O_RDWR | O_BINARY, 0664);
if (fFd >= 0)
{
#ifndef _MSC_VER
//FIXME:
fchmod(fFd, 0666); // XXXPAT: override umask at least for testing
#endif
try
{
initializeBitmap();
}
catch (...)
{
close(fFd);
throw;
}
}
else if (errno == EEXIST)
{
fFd = open(fFilename.c_str(), O_RDWR | O_BINARY);
if (fFd < 0)
{
os << "Couldn't open oid bitmap file " << fFilename << ": " <<
strerror(errno);
log(os.str());
throw ios_base::failure(os.str());
}
}
else
{
os << "Couldn't create oid bitmap file " << fFilename << ": " <<
strerror(errno);
log(os.str());
throw ios_base::failure(os.str());
}
}
loadVBOIDs();
}
@ -622,7 +455,6 @@ retry:
{
writeData(buf, offset, byteSize);
if (true || IDBPolicy::useHdfs())
fFp->flush();
delete [] buf;
@ -669,7 +501,6 @@ retry:
{
writeData(buf, offset, byteSize);
if (true || IDBPolicy::useHdfs())
fFp->flush();
delete [] buf;
@ -790,8 +621,6 @@ void OIDServer::patchFreelist(struct FEntry* freelist, int start, int num) const
if (changed)
{
writeData(reinterpret_cast<uint8_t*>(freelist), 0, HeaderSize);
if (true || IDBPolicy::useHdfs())
fFp->flush();
}
}
@ -819,7 +648,6 @@ int OIDServer::allocVBOID(uint16_t dbroot)
throw;
}
if (true || IDBPolicy::useHdfs())
fFp->flush();
return ret;
@ -893,7 +721,6 @@ int OIDServer::allocOIDs(int num)
writeData(reinterpret_cast<uint8_t*>(freelist), 0, HeaderSize);
flipOIDBlock(bestMatchBegin, num, 0);
if (true || IDBPolicy::useHdfs())
fFp->flush();
return bestMatchBegin;

114
versioning/BRM/sessionmanagerserver.cpp
View File

@ -85,7 +85,7 @@ const uint32_t SessionManagerServer::SS_FORCE = 1 << 5; // In combination w
const uint32_t SessionManagerServer::SS_QUERY_READY = 1 << 6; // Set by ProcManager when system is ready for queries
SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0), txnidfd(-1)
SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0)
{
config::Config* conf;
string stmp;
@ -119,26 +119,6 @@ SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0), txnidfd
txnidFilename = conf->getConfig("SessionManager", "TxnIDFile");
if (false && !IDBPolicy::useHdfs())
{
txnidfd = open(txnidFilename.c_str(), O_RDWR | O_CREAT | O_BINARY, 0664);
if (txnidfd < 0)
{
perror("SessionManagerServer(): open");
throw runtime_error("SessionManagerServer: Could not open the transaction ID file");
}
//FIXME: do we need this on Win?
#ifndef _MSC_VER
else
{
fchmod(txnidfd, 0664);
}
#endif
}
semValue = maxTxns;
_verID = 0;
_sysCatVerID = 0;
@ -154,6 +134,9 @@ SessionManagerServer::SessionManagerServer() : unique32(0), unique64(0), txnidfd
}
}
SessionManagerServer::~SessionManagerServer()
{
}
void SessionManagerServer::reset()
{
mutex.try_lock();
@ -178,57 +161,7 @@ again:
// If we fail to read a full four bytes for any value, then the
// value isn't in the file, and we start with the default.
if (false && !IDBPolicy::useHdfs())
{
// Last transaction id
lseek(txnidfd, 0, SEEK_SET);
err = read(txnidfd, &lastTxnID, 4);
if (err < 0 && errno != EINTR)
{
perror("Sessionmanager::initSegment(): read");
throw runtime_error("SessionManagerServer: read failed, aborting");
}
else if (err < 0)
goto again;
else if (err == sizeof(int))
_verID = lastTxnID;
// last system catalog version id
err = read(txnidfd, &lastSysCatVerId, 4);
if (err < 0 && errno != EINTR)
{
perror("Sessionmanager::initSegment(): read");
throw runtime_error("SessionManagerServer: read failed, aborting");
}
else if (err < 0)
goto again;
else if (err == sizeof(int))
_sysCatVerID = lastSysCatVerId;
// System state. Contains flags regarding the suspend state of the system.
err = read(txnidfd, &systemState, 4);
if (err < 0 && errno == EINTR)
{
goto again;
}
else if (err == sizeof(int))
{
// Turn off the pending and force flags. They make no sense for a clean start.
// Turn off the ready flag. DMLProc will set it back on when
// initialized.
systemState &=
~(SS_READY | SS_QUERY_READY | SS_SUSPEND_PENDING | SS_SHUTDOWN_PENDING | SS_ROLLBACK | SS_FORCE);
}
else
{
// else no problem. System state wasn't saved. Might be an upgraded system.
systemState = 0;
}
}
else if (IDBPolicy::exists(txnidFilename.c_str()))
if (IDBPolicy::exists(txnidFilename.c_str()))
{
scoped_ptr<IDBDataFile> txnidfp(IDBDataFile::open(
IDBPolicy::getType(txnidFilename.c_str(),
@ -296,28 +229,9 @@ again:
* word of txnid File
*/
void SessionManagerServer::saveSystemState()
{
if (false && !IDBPolicy::useHdfs())
{
int err = 0;
uint32_t lSystemState = systemState;
// We don't save the pending flags, the force flag or the ready flags.
lSystemState &= ~(SS_READY | SS_QUERY_READY | SS_SUSPEND_PENDING | SS_SHUTDOWN_PENDING | SS_FORCE);
lseek(txnidfd, 8, SEEK_SET);
err = write(txnidfd, &lSystemState, sizeof(int));
if (err < 0)
{
perror("SessionManagerServer::saveSystemState(): write(systemState)");
throw runtime_error("SessionManagerServer::saveSystemState(): write(systemState) failed");
}
}
else
{
saveSMTxnIDAndState();
}
}
const QueryContext SessionManagerServer::verID()
{
@ -377,25 +291,7 @@ const TxnID SessionManagerServer::newTxnID(const SID session, bool block, bool i
if (isDDL)
++_sysCatVerID;
if (false && !IDBPolicy::useHdfs())
{
int filedata[2];
filedata[0] = _verID;
filedata[1] = _sysCatVerID;
lseek(txnidfd, 0, SEEK_SET);
int err = write(txnidfd, filedata, 8);
if (err < 0)
{
perror("SessionManagerServer::newTxnID(): write(verid)");
throw runtime_error("SessionManagerServer::newTxnID(): write(verid) failed");
}
}
else
{
saveSMTxnIDAndState();
}
return ret;
}

6
versioning/BRM/sessionmanagerserver.h
View File

@ -123,10 +123,7 @@ public:
* It does not destroy the semaphores. Those persist until the system
* is shut down.
*/
virtual ~SessionManagerServer()
{
if (txnidfd >= 0 ) close(txnidfd);
}
virtual ~SessionManagerServer();
/** @brief Gets the current version ID
*
@ -276,7 +273,6 @@ private:
int maxTxns; // the maximum number of concurrent transactions
std::string txnidFilename;
int txnidfd; // file descriptor for the "last txnid" file
execplan::CalpontSystemCatalog::SCN _verID;
execplan::CalpontSystemCatalog::SCN _sysCatVerID;
uint32_t systemState;

134
versioning/BRM/slavecomm.cpp
View File

@ -65,7 +65,7 @@ namespace BRM
{
SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) :
slave(s), currentSaveFD(-1), currentSaveFile(NULL), journalh(NULL)
slave(s), currentSaveFile(NULL), journalh(NULL)
#ifdef _MSC_VER
, fPids(0), fMaxPids(64)
#endif
@ -138,21 +138,10 @@ SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) :
firstSlave = true;
journalName = savefile + "_journal";
const char* filename = journalName.c_str();
uint32_t utmp = ::umask(0);
if (true || IDBPolicy::useHdfs())
{
journalh = IDBDataFile::open(
IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "a", 0);
}
else
{
journal.open(filename, ios_base::binary | ios_base::out | ios_base::app);
}
::umask(utmp);
if ((journal.is_open() == false) && (journalh == NULL))
if (journalh == NULL)
throw runtime_error("Could not open the BRM journal for writing!");
}
else
@ -180,7 +169,7 @@ SlaveComm::SlaveComm(string hostname, SlaveDBRMNode* s) :
}
SlaveComm::SlaveComm()
: currentSaveFD(-1), currentSaveFile(NULL), journalh(NULL)
: currentSaveFile(NULL), journalh(NULL)
#ifdef _MSC_VER
, fPids(0), fMaxPids(64)
#endif
@ -222,7 +211,6 @@ SlaveComm::~SlaveComm()
if (firstSlave)
{
close(currentSaveFD);
delete currentSaveFile;
currentSaveFile = NULL;
}
@ -2007,19 +1995,13 @@ void SlaveComm::do_confirm()
if (firstSlave && (takeSnapshot ||
(journalCount >= snapshotInterval && snapshotInterval >= 0)))
{
const char* filename = tmp.c_str();
if (false && !IDBPolicy::useHdfs() && currentSaveFD < 0)
{
currentSaveFD = open(filename, O_WRONLY | O_CREAT, 0664);
}
else if ((true || IDBPolicy::useHdfs()) && !currentSaveFile)
if (!currentSaveFile)
{
currentSaveFile = IDBDataFile::open(
IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "wb", 0);
IDBPolicy::getType(tmp.c_str(), IDBPolicy::WRITEENG), tmp.c_str(), "wb", 0);
}
if (currentSaveFD < 0 && currentSaveFile == NULL)
if (currentSaveFile == NULL)
{
ostringstream os;
os << "WorkerComm: failed to open the current savefile. errno: "
@ -2028,9 +2010,7 @@ void SlaveComm::do_confirm()
throw runtime_error(os.str());
}
#ifndef _MSC_VER
chmod(filename, 0664);
#endif
tmp = savefile + (saveFileToggle ? 'A' : 'B');
slave->saveState(tmp);
#ifndef _MSC_VER
@ -2038,8 +2018,6 @@ void SlaveComm::do_confirm()
#endif
int err = 0;
if (currentSaveFile)
{
// MCOL-1558. Make the _current file relative to DBRMRoot.
string relative = tmp.substr(tmp.find_last_of('/') + 1);
err = currentSaveFile->write(relative.c_str(), relative.length());
@ -2061,51 +2039,12 @@ void SlaveComm::do_confirm()
currentSaveFile = NULL;
saveFileToggle = !saveFileToggle;
const char* filename = journalName.c_str();
//uint32_t utmp = ::umask(0);
delete journalh;
journalh = IDBDataFile::open(
IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "w+b", 0);
//::umask(utmp);
IDBPolicy::getType(journalName.c_str(), IDBPolicy::WRITEENG), journalName.c_str(), "w+b", 0);
if (!journalh)
throw runtime_error("Could not open the BRM journal for writing!");
}
else
{
lseek(currentSaveFD, 0, SEEK_SET);
// MCOL-1558. Make the _current file relative to DBRMRoot.
string relative = tmp.substr(tmp.find_last_of('/') + 1);
err = write(currentSaveFD, relative.c_str(), relative.length());
if (err < (int) relative.length())
{
ostringstream os;
os << "WorkerComm: currentfile write() returned " << err
<< " fd is " << currentSaveFD;
if (err < 0)
os << " errno: " << strerror(errno);
log(os.str());
}
#ifdef _MSC_VER
//FIXME: Do we need to account for Windows EOL conversions?
_chsize_s(currentSaveFD, tmp.length());
_commit(currentSaveFD);
#else
err = ftruncate(currentSaveFD, relative.length());
fsync(currentSaveFD);
#endif
saveFileToggle = !saveFileToggle;
/* Is there a nicer way to truncate the file using an ofstream? */
journal.close();
uint32_t utmp = ::umask(0);
journal.open(journalName.c_str(), ios_base::binary | ios_base::out | ios_base::trunc);
::umask(utmp);
}
takeSnapshot = false;
doSaveDelta = false;
@ -2258,8 +2197,6 @@ int SlaveComm::replayJournal(string prefix)
const char* filename = journalName.c_str();
if (true || IDBPolicy::useHdfs())
{
IDBDataFile* journalf = IDBDataFile::open(
IDBPolicy::getType(filename, IDBPolicy::WRITEENG), filename, "rb", 0);
@ -2280,6 +2217,7 @@ int SlaveComm::replayJournal(string prefix)
if (readIn > 0)
{
cmd.needAtLeast(len);
readIn = journalf->read((char*) cmd.getInputPtr(), len);
cmd.advanceInputPtr(len);
@ -2299,47 +2237,7 @@ int SlaveComm::replayJournal(string prefix)
cmd.restart();
ret++;
}
}
while (readIn > 0);
}
else
{
ifstream journalf;
journalf.open(filename, ios_base::in | ios_base::binary);
if (!journalf.is_open())
{
cout << "Error opening journal file " << fName << endl;
return -1;
}
while (journalf)
{
journalf.read((char*) &len, sizeof(len));
if (!journalf)
break;
cmd.needAtLeast(len);
journalf.read((char*) cmd.getInputPtr(), len);
cmd.advanceInputPtr(len);
try
{
processCommand(cmd);
}
catch (exception& e)
{
cout << e.what() << " Journal replay was incomplete." << endl;
slave->undoChanges();
return -1;
}
slave->confirmChanges();
cmd.restart();
ret++;
}
}
} while (readIn > 0);
return ret;
}
@ -2368,20 +2266,10 @@ void SlaveComm::saveDelta()
{
uint32_t len = delta.length();
if (true || IDBPolicy::useHdfs())
{
journalh->write((const char*) &len, sizeof(len));
journalh->write((const char*) delta.buf(), delta.length());
journalh->flush();
}
else
{
journal.seekg(0, ios_base::end);
journal.write((const char*) &len, sizeof(len));
journal.write((const char*) delta.buf(), delta.length());
journal.sync();
}
journalCount++;
}
catch (exception& e)

2
versioning/BRM/slavecomm.h
View File

@ -125,10 +125,8 @@ private:
std::string savefile;
bool release, die, firstSlave, saveFileToggle, takeSnapshot, doSaveDelta, standalone, printOnly;
messageqcpp::ByteStream delta;
int currentSaveFD;
idbdatafile::IDBDataFile* currentSaveFile;
std::string journalName;
std::fstream journal;
idbdatafile::IDBDataFile* journalh;
int64_t snapshotInterval, journalCount;
struct timespec MSG_TIMEOUT;

63
versioning/BRM/tablelockserver.cpp
View File

@ -63,8 +63,6 @@ void TableLockServer::save()
const char* filename_p = filename.c_str();
if (true || IDBPolicy::useHdfs())
{
scoped_ptr<IDBDataFile> out(IDBDataFile::open(
IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
filename_p, "wb", 0));
@ -82,24 +80,6 @@ void TableLockServer::save()
it->second.serialize(out.get());
}
}
else
{
ofstream out(filename.c_str(), ios::trunc | ios::binary | ios::out );
if (!out)
throw runtime_error("TableLockServer::save(): could not open save file");
out.write((char*) &count, 4);
for (it = locks.begin(); it != locks.end(); ++it)
{
if (!out)
throw runtime_error("TableLockServer::save(): could not write save file");
it->second.serialize(out);
}
}
}
// call with lock held
void TableLockServer::load()
@ -109,8 +89,6 @@ void TableLockServer::load()
TableLockInfo tli;
/* Need to standardize the file error handling */
if (true || IDBPolicy::useHdfs())
{
const char* filename_p = filename.c_str();
scoped_ptr<IDBDataFile> in(IDBDataFile::open(
IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
@ -149,47 +127,6 @@ void TableLockServer::load()
throw;
}
}
else
{
ifstream in(filename.c_str(), ios::binary | ios::in);
if (!in)
{
ostringstream os;
os << "TableLockServer::load(): could not open the save file"
<< filename;
log(os.str(), logging::LOG_TYPE_DEBUG);
return;
}
in.exceptions(ios::failbit | ios::badbit);
try
{
in.read((char*) &size, 4);
for (i = 0; i < size; i++)
{
tli.deserialize(in);
tli.id = sms->getUnique64(); // Need new #s...
if (tli.id == 0) // 0 is an error code
tli.id = sms->getUnique64();
locks[tli.id] = tli;
}
}
catch (std::exception& e)
{
ostringstream os;
os << "TableLockServer::load(): could not load save file " << filename <<
" loaded " << i << "/" << size << " entries\n";
log(os.str(), logging::LOG_TYPE_DEBUG);
throw;
}
}
}
// throws on a failed save()

110
versioning/BRM/vbbm.cpp
View File

@ -21,7 +21,6 @@
****************************************************************************/
#include <iostream>
#include <fstream>
#include <vector>
#include <sstream>
#include <sys/types.h>
@ -980,54 +979,6 @@ void VBBM::setReadOnly()
* struct VBBMEntry * numEntries
*/
void VBBM::loadVersion1(IDBDataFile* in)
{
int vbbmEntries, i;
VBBMEntry entry;
clear();
if (in->read((char*) &vbbmEntries, 4) != 4)
{
log_errno("VBBM::load()");
throw runtime_error("VBBM::load(): Failed to read entry number");
}
for (i = 0; i < vbbmEntries; i++)
{
if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry))
{
log_errno("VBBM::load()");
throw runtime_error("VBBM::load(): Failed to load entry");
}
insert(entry.lbid, entry.verID, entry.vbOID, entry.vbFBO, true);
//confirmChanges();
addVBFileIfNotExists(entry.vbOID);
}
/* This will load the saved file data from 2.2, but it is not compatible with
* 3.0+. If enabled, take out the addVBFile..() call above
*/
#if 0
int dummy, nFiles;
in.read((char*) &nFiles, 4);
cout << "got nfiles = " << nFiles << endl;
in.read((char*) &dummy, 4); // an unused var in 3.0+
while (vbbm->nFiles < nFiles)
growVBBM(true); // this allocates one file, doesn't grow the main storage
in.read((char*) files, sizeof(VBFileMetadata) * nFiles);
for (i = 0; i < nFiles; i++)
cout << "file " << i << ": oid=" << files[i].OID << " size=" << files[i].fileSize
<< " offset=" << files[i].nextOffset << endl;
#endif
}
void VBBM::loadVersion2(IDBDataFile* in)
{
@ -1092,19 +1043,6 @@ void VBBM::loadVersion2(IDBDataFile* in)
insert(loadedEntries[i].lbid, loadedEntries[i].verID, loadedEntries[i].vbOID,
loadedEntries[i].vbFBO, true);
/*
for (i = 0; i < vbbmEntries; i++)
{
if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry))
{
log_errno("VBBM::load()");
throw runtime_error("VBBM::load(): Failed to load entry");
}
insert(entry.lbid, entry.verID, entry.vbOID, entry.vbFBO, true);
}
*/
}
//#include "boost/date_time/posix_time/posix_time.hpp"
@ -1138,9 +1076,6 @@ void VBBM::load(string filename)
switch (magic)
{
case VBBM_MAGIC_V1:
loadVersion1(in.get());
break;
case VBBM_MAGIC_V2:
loadVersion2(in.get());
@ -1161,9 +1096,6 @@ void VBBM::save(string filename)
int i;
int var;
// XXXPAT: forcing the IDB* path. Delete the fstream path when appropriate.
if (true || IDBPolicy::useHdfs())
{
const char* filename_p = filename.c_str();
scoped_ptr<IDBDataFile> out(IDBDataFile::open(
IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
@ -1210,6 +1142,7 @@ void VBBM::save(string filename)
}
first = -1;
}
}
if (first != -1)
{
@ -1228,47 +1161,6 @@ void VBBM::save(string filename)
}
}
/*
for (i = 0; i < vbbm->vbCapacity; i++)
{
if (storage[i].lbid != -1)
{
bytesToWrite += sizeof(VBBMEntry);
bytesWritten += out->write((char*)&storage[i], sizeof(VBBMEntry));
}
}
if (bytesWritten != bytesToWrite)
{
log_errno("VBBM::save()");
throw runtime_error("VBBM::save(): Failed to write the file");
}
*/
}
else
{
ofstream out;
out.open(filename.c_str(), ios_base::trunc | ios_base::out | ios_base::binary);
if (!out)
{
log_errno("VBBM::save()");
throw runtime_error("VBBM::save(): Failed to open the file");
}
out.exceptions(ios_base::badbit);
var = VBBM_MAGIC_V2;
out.write((char*) &var, 4);
out.write((char*) &vbbm->vbCurrentSize, 4);
out.write((char*) &vbbm->nFiles, 4);
out.write((char*) files, sizeof(VBFileMetadata) * vbbm->nFiles);
for (i = 0; i < vbbm->vbCapacity; i++)
if (storage[i].lbid != -1)
out.write((char*)&storage[i], sizeof(VBBMEntry));
}
#if 0
cout << "saving... nfiles=" << vbbm->nFiles << "\n";

1
versioning/BRM/vbbm.h
View File

@ -214,7 +214,6 @@ public:
EXPORT void clear();
EXPORT void load(std::string filename);
EXPORT void loadVersion1(idbdatafile::IDBDataFile* in);
EXPORT void loadVersion2(idbdatafile::IDBDataFile* in);
EXPORT void save(std::string filename);

135
versioning/BRM/vss.cpp
View File

@ -21,7 +21,6 @@
****************************************************************************/
#include <iostream>
#include <fstream>
#include <sstream>
//#define NDEBUG
#include <cassert>
@ -1294,9 +1293,6 @@ void VSS::save(string filename)
int i;
struct Header header;
// XXXPAT: Forcing the IDB* path to run. Delete the fstream path when appropriate.
if (true || IDBPolicy::useHdfs())
{
const char* filename_p = filename.c_str();
scoped_ptr<IDBDataFile> out(IDBDataFile::open(
IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
@ -1359,54 +1355,6 @@ void VSS::save(string filename)
}
}
/*
for (i = 0; i < vss->capacity; i++)
{
if (storage[i].lbid != -1)
{
if (out->write((char*)&storage[i], sizeof(VSSEntry)) != sizeof(VSSEntry))
{
log_errno("VSS::save()");
throw runtime_error("VSS::save(): Failed to write vss entry to the file");
}
}
}
*/
}
else
{
ofstream out;
out.open(filename.c_str(), ios_base::trunc | ios_base::out | ios_base::binary);
//::umask(utmp);
if (!out)
{
log_errno("VSS::save()");
throw runtime_error("VSS::save(): Failed to open the file");
}
out.exceptions(ios_base::badbit);
header.magic = VSS_MAGIC_V1;
header.entries = vss->currentSize;
try
{
out.write((char*)&header, sizeof(header));
for (i = 0; i < vss->capacity; i++)
if (storage[i].lbid != -1)
out.write((char*)&storage[i], sizeof(VSSEntry));
}
catch (std::exception& e)
{
out.close();
throw;
}
out.close();
}
}
// Ideally, we;d like to get in and out of this fcn as quickly as possible.
@ -1518,93 +1466,10 @@ void VSS::load(string filename)
insert(loadedEntries[i].lbid, loadedEntries[i].verID, loadedEntries[i].vbFlag,
loadedEntries[i].locked, true);
/*
for (i = 0; i < header.entries; i++)
{
if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry))
{
log_errno("VSS::load()");
throw runtime_error("VSS::load(): Failed to read entry");
}
insert(entry.lbid, entry.verID, entry.vbFlag, entry.locked, true);
}
*/
//time2 = microsec_clock::local_time();
//cout << "done loading " << time2 << " duration: " << time2-time1 << endl;
}
#ifndef __LP64__
//This code is OBE now that the structs are padded correctly
struct VSSEntry_
{
LBID_t lbid;
VER_t verID;
bool vbFlag;
bool locked;
int next;
uint32_t pad1;
};
void VSS::load64(string filename)
{
int i;
struct Header header;
struct VSSEntry_ entry;
const char* filename_p = filename.c_str();
scoped_ptr<IDBDataFile> in(IDBDataFile::open(
IDBPolicy::getType(filename_p, IDBPolicy::WRITEENG),
filename_p, "rb", 0));
if (!in)
{
log_errno("VSS::load()");
throw runtime_error("VSS::load(): Failed to open the file");
}
if (in->read((char*)&header, sizeof(header)) != sizeof(header))
{
log_errno("VSS::load()");
throw runtime_error("VSS::load(): Failed to read header");
}
if (header.magic != VSS_MAGIC_V1)
{
log("VSS::load(): Bad magic. Not a VSS file?");
throw runtime_error("VSS::load(): Bad magic. Not a VSS file?");
}
if (header.entries < 0)
{
log("VSS::load(): Bad size. Not a VSS file?");
throw runtime_error("VSS::load(): Bad size. Not a VSS file?");
}
for (i = 0; i < vss->capacity; i++)
storage[i].lbid = -1;
for (i = 0; i < vss->numHashBuckets; i++)
hashBuckets[i] = -1;
vss->currentSize = 0;
vss->lockedEntryCount = 0;
vss->LWM = 0;
for (i = 0; i < header.entries; i++)
{
if (in->read((char*)&entry, sizeof(entry)) != sizeof(entry))
{
log_errno("VSS::load()");
throw runtime_error("VSS::load(): Failed to read entry");
}
insert(entry.lbid, entry.verID, entry.vbFlag, entry.locked, true);
}
}
#endif
#ifdef BRM_DEBUG
// read lock

4
versioning/BRM/vss.h
View File

@ -250,10 +250,6 @@ public:
EXPORT void clear();
EXPORT void load(std::string filename);
#ifndef __LP64__
//This method is OBE now that the structs are padded correctly
EXPORT void load64(std::string filename);
#endif
EXPORT void save(std::string filename);
#ifdef BRM_DEBUG