database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

feat(PP,ByteStream): new counting memory allocator

Browse Source
This commit is contained in:
drrtuy
2024-11-22 00:56:26 +00:00
parent 2d69b49ba0
commit 02b8ea1331
27 changed files with 548 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
primitives/primproc/batchprimitiveprocessor.cpp
View File

@ -213,13 +213,6 @@ BatchPrimitiveProcessor::BatchPrimitiveProcessor(ByteStream& b, double prefetch,
initBPP(b);
}
#if 0
BatchPrimitiveProcessor::BatchPrimitiveProcessor(const BatchPrimitiveProcessor& bpp)
{
throw logic_error("copy BPP deprecated");
}
#endif
BatchPrimitiveProcessor::~BatchPrimitiveProcessor()
{
// FIXME: just do a sync fetch
@ -247,6 +240,8 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs)
uint8_t tmp8;
uint16_t tmp16;
Command::CommandType type;
auto cnt = exemgr::globServiceExeMgr->getRm().availableMemory();
std::cout << "initBPP availableMemory: " << cnt << std::endl;
bs.advance(sizeof(ISMPacketHeader)); // skip the header
bs >> tmp8;
@ -365,13 +360,17 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs)
if (!typelessJoin[i])
{
auto alloc = exemgr::globServiceExeMgr->getRm().getAllocator<TJoiner::value_type>();
bs >> joinNullValues[i];
bs >> largeSideKeyColumns[i];
for (uint j = 0; j < processorThreads; ++j)
tJoiners[i][j].reset(new TJoiner(10, TupleJoiner::hasher()));
tJoiners[i][j].reset(new TJoiner(10, TupleJoiner::hasher(), alloc));
}
else
{
auto alloc = exemgr::globServiceExeMgr->getRm().getAllocator<TLJoiner::value_type>();
deserializeVector<uint32_t>(bs, tlLargeSideKeyColumns[i]);
bs >> tlSmallSideKeyLengths[i];
bs >> tmp8;
@ -393,7 +392,7 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs)
mSmallSideKeyColumnsPtr, mSmallSideRGPtr);
auto tlComparator = TupleJoiner::TypelessDataComparator(&outputRG, &tlLargeSideKeyColumns[i],
mSmallSideKeyColumnsPtr, mSmallSideRGPtr);
tlJoiners[i][j].reset(new TLJoiner(10, tlHasher, tlComparator));
tlJoiners[i][j].reset(new TLJoiner(10, tlHasher, tlComparator, alloc));
}
}
}
@ -497,7 +496,7 @@ void BatchPrimitiveProcessor::initBPP(ByteStream& bs)
bs >> *(fAggregator.get());
// If there's UDAF involved, set up for PM processing
for (const auto & pcol : fAggregator->getAggFunctions())
for (const auto& pcol : fAggregator->getAggFunctions())
{
auto* rowUDAF = dynamic_cast<RowUDAFFunctionCol*>(pcol.get());
@ -843,6 +842,8 @@ int BatchPrimitiveProcessor::endOfJoiner()
{
endOfJoinerRan = true;
pthread_mutex_unlock(&objLock);
auto cnt = exemgr::globServiceExeMgr->getRm().availableMemory();
std::cout << "endOfJoiner availableMemory: " << cnt << std::endl;
return 0;
}
@ -885,6 +886,8 @@ int BatchPrimitiveProcessor::endOfJoiner()
endOfJoinerRan = true;
pthread_mutex_unlock(&objLock);
auto cnt = exemgr::globServiceExeMgr->getRm().availableMemory();
std::cout << "endOfJoiner availableMemory: " << cnt << std::endl;
return 0;
}
@ -1218,7 +1221,7 @@ uint32_t BatchPrimitiveProcessor::executeTupleJoin(uint32_t startRid, RowGroup&
{
bool hasNull = false;
for (unsigned int column: tlLargeSideKeyColumns[j])
for (unsigned int column : tlLargeSideKeyColumns[j])
if (oldRow.isNullValue(column))
{
hasNull = true;
@ -1374,7 +1377,7 @@ uint32_t BatchPrimitiveProcessor::executeTupleJoin(uint32_t startRid, RowGroup&
#ifdef PRIMPROC_STOPWATCH
void BatchPrimitiveProcessor::execute(StopWatch* stopwatch)
#else
void BatchPrimitiveProcessor::execute()
void BatchPrimitiveProcessor::execute(messageqcpp::SBS& bs)
#endif
{
uint8_t sendCount = 0;
@ -1509,7 +1512,7 @@ void BatchPrimitiveProcessor::execute()
writeProjectionPreamble();
stopwatch->stop("- writeProjectionPreamble");
#else
writeProjectionPreamble();
writeProjectionPreamble(bs);
#endif
}
@ -1536,7 +1539,7 @@ void BatchPrimitiveProcessor::execute()
{
for (j = 0; j < projectCount; ++j)
{
projectSteps[j]->project();
projectSteps[j]->project(bs);
}
}
else
@ -1641,9 +1644,9 @@ void BatchPrimitiveProcessor::execute()
if (!fAggregator)
{
*serialized << (uint8_t)1; // the "count this msg" var
*bs << (uint8_t)1; // the "count this msg" var
fe2Output.setDBRoot(dbRoot);
fe2Output.serializeRGData(*serialized);
fe2Output.serializeRGData(*bs);
//*serialized << fe2Output.getDataSize();
// serialized->append(fe2Output.getData(), fe2Output.getDataSize());
}
@ -1653,7 +1656,7 @@ void BatchPrimitiveProcessor::execute()
{
utils::setThreadName("BPPAgg_1");
*serialized << (uint8_t)1; // the "count this msg" var
*bs << (uint8_t)1; // the "count this msg" var
// see TupleBPS::setFcnExpGroup2() and where it gets called.
// it sets fe2 there, on the other side of communication.
@ -1669,25 +1672,25 @@ void BatchPrimitiveProcessor::execute()
if ((currentBlockOffset + 1) == count) // @bug4507, 8k
{
fAggregator->loadResult(*serialized); // @bug4507, 8k
fAggregator->loadResult(*bs); // @bug4507, 8k
} // @bug4507, 8k
else if (utils::MonitorProcMem::isMemAvailable()) // @bug4507, 8k
{
fAggregator->loadEmptySet(*serialized); // @bug4507, 8k
fAggregator->loadEmptySet(*bs); // @bug4507, 8k
} // @bug4507, 8k
else // @bug4507, 8k
{
fAggregator->loadResult(*serialized); // @bug4507, 8k
fAggregator->aggReset(); // @bug4507, 8k
fAggregator->loadResult(*bs); // @bug4507, 8k
fAggregator->aggReset(); // @bug4507, 8k
} // @bug4507, 8k
}
if (!fAggregator && !fe2)
{
*serialized << (uint8_t)1; // the "count this msg" var
*bs << (uint8_t)1; // the "count this msg" var
outputRG.setDBRoot(dbRoot);
// cerr << "serializing " << outputRG.toString() << endl;
outputRG.serializeRGData(*serialized);
outputRG.serializeRGData(*bs);
//*serialized << outputRG.getDataSize();
// serialized->append(outputRG.getData(), outputRG.getDataSize());
@ -1700,7 +1703,7 @@ void BatchPrimitiveProcessor::execute()
else // Is doJoin
{
uint32_t startRid = 0;
ByteStream preamble = *serialized;
ByteStream preamble = *bs;
origRidCount = ridCount; // ridCount can get modified by executeTupleJoin(). We need to keep track of
// the original val.
/* project the key columns. If there's the filter IN the join, project everything.
@ -1783,7 +1786,7 @@ void BatchPrimitiveProcessor::execute()
sendCount = (uint8_t)(!moreRGs && !startRid);
// *serialized << (uint8_t)(!moreRGs && !startRid); // the "count
// this msg" var
*serialized << sendCount;
*bs << sendCount;
if (fe2)
{
utils::setThreadName("BPPFE2_2");
@ -1817,30 +1820,30 @@ void BatchPrimitiveProcessor::execute()
if ((currentBlockOffset + 1) == count && moreRGs == false && startRid == 0) // @bug4507, 8k
{
fAggregator->loadResult(*serialized); // @bug4507, 8k
fAggregator->loadResult(*bs); // @bug4507, 8k
} // @bug4507, 8k
else if (utils::MonitorProcMem::isMemAvailable()) // @bug4507, 8k
{
fAggregator->loadEmptySet(*serialized); // @bug4507, 8k
fAggregator->loadEmptySet(*bs); // @bug4507, 8k
} // @bug4507, 8k
else // @bug4507, 8k
{
fAggregator->loadResult(*serialized); // @bug4507, 8k
fAggregator->aggReset(); // @bug4507, 8k
fAggregator->loadResult(*bs); // @bug4507, 8k
fAggregator->aggReset(); // @bug4507, 8k
} // @bug4507, 8k
}
else
{
// cerr <<" * serialzing " << nextRG.toString() << endl;
nextRG.serializeRGData(*serialized);
nextRG.serializeRGData(*bs);
}
/* send the msg & reinit the BS */
if (moreRGs)
{
sendResponse();
serialized.reset(new ByteStream());
*serialized = preamble;
sendResponse(bs);
bs.reset(new ByteStream());
*bs = preamble;
}
}
@ -1848,16 +1851,16 @@ void BatchPrimitiveProcessor::execute()
{
// Should we happen to finish sending data rows right on the boundary of when moreRGs flips off,
// then we need to start a new buffer. I.e., it needs the count this message byte pushed.
if (serialized->length() == preamble.length())
*serialized << (uint8_t)(startRid > 0 ? 0 : 1); // the "count this msg" var
if (bs->length() == preamble.length())
*bs << (uint8_t)(startRid > 0 ? 0 : 1); // the "count this msg" var
*serialized << ridCount;
*bs << ridCount;
for (i = 0; i < joinerCount; i++)
{
for (j = 0; j < ridCount; ++j)
{
serializeInlineVector<uint32_t>(*serialized, tSmallSideMatches[i][j]);
serializeInlineVector<uint32_t>(*bs, tSmallSideMatches[i][j]);
tSmallSideMatches[i][j].clear();
}
}
@ -1872,10 +1875,10 @@ void BatchPrimitiveProcessor::execute()
}
else
{
*serialized << (uint8_t)(startRid > 0 ? 0 : 1); // the "count this msg" var
*bs << (uint8_t)(startRid > 0 ? 0 : 1); // the "count this msg" var
outputRG.setDBRoot(dbRoot);
// cerr << "serializing " << outputRG.toString() << endl;
outputRG.serializeRGData(*serialized);
outputRG.serializeRGData(*bs);
//*serialized << outputRG.getDataSize();
// serialized->append(outputRG.getData(), outputRG.getDataSize());
@ -1883,16 +1886,16 @@ void BatchPrimitiveProcessor::execute()
{
for (j = 0; j < ridCount; ++j)
{
serializeInlineVector<uint32_t>(*serialized, tSmallSideMatches[i][j]);
serializeInlineVector<uint32_t>(*bs, tSmallSideMatches[i][j]);
tSmallSideMatches[i][j].clear();
}
}
}
if (startRid > 0)
{
sendResponse();
serialized.reset(new ByteStream());
*serialized = preamble;
sendResponse(bs);
bs.reset(new ByteStream());
*bs = preamble;
}
} while (startRid > 0);
}
@ -1905,11 +1908,11 @@ void BatchPrimitiveProcessor::execute()
// sendCount << std::endl;
if (projectCount > 0 || ot == ROW_GROUP)
{
*serialized << cachedIO;
*bs << cachedIO;
cachedIO = 0;
*serialized << physIO;
*bs << physIO;
physIO = 0;
*serialized << touchedBlocks;
*bs << touchedBlocks;
touchedBlocks = 0;
// cout << "sent physIO=" << physIO << " cachedIO=" << cachedIO <<
// " touchedBlocks=" << touchedBlocks << endl;
@ -1922,15 +1925,15 @@ void BatchPrimitiveProcessor::execute()
}
catch (logging::QueryDataExcept& qex)
{
writeErrorMsg(qex.what(), qex.errorCode());
writeErrorMsg(bs, qex.what(), qex.errorCode());
}
catch (logging::DictionaryBufferOverflow& db)
{
writeErrorMsg(db.what(), db.errorCode());
writeErrorMsg(bs, db.what(), db.errorCode());
}
catch (scalar_exception& se)
{
writeErrorMsg(IDBErrorInfo::instance()->errorMsg(ERR_MORE_THAN_1_ROW), ERR_MORE_THAN_1_ROW, false);
writeErrorMsg(bs, IDBErrorInfo::instance()->errorMsg(ERR_MORE_THAN_1_ROW), ERR_MORE_THAN_1_ROW, false);
}
catch (NeedToRestartJob& n)
{
@ -1941,20 +1944,21 @@ void BatchPrimitiveProcessor::execute()
}
catch (IDBExcept& iex)
{
writeErrorMsg(iex.what(), iex.errorCode(), true, false);
writeErrorMsg(bs, iex.what(), iex.errorCode(), true, false);
}
catch (const std::exception& ex)
{
writeErrorMsg(ex.what(), logging::batchPrimitiveProcessorErr);
writeErrorMsg(bs, ex.what(), logging::batchPrimitiveProcessorErr);
}
catch (...)
{
string msg("BatchPrimitiveProcessor caught an unknown exception");
writeErrorMsg(msg, logging::batchPrimitiveProcessorErr);
writeErrorMsg(bs, msg, logging::batchPrimitiveProcessorErr);
}
}
void BatchPrimitiveProcessor::writeErrorMsg(const string& error, uint16_t errCode, bool logIt, bool critical)
void BatchPrimitiveProcessor::writeErrorMsg(messageqcpp::SBS& bs, const string& error, uint16_t errCode,
bool logIt, bool critical)
{
ISMPacketHeader ism;
PrimitiveHeader ph;
@ -1970,10 +1974,10 @@ void BatchPrimitiveProcessor::writeErrorMsg(const string& error, uint16_t errCod
ph.UniqueID = uniqueID;
ism.Status = errCode;
serialized.reset(new ByteStream());
serialized->append((uint8_t*)&ism, sizeof(ism));
serialized->append((uint8_t*)&ph, sizeof(ph));
*serialized << error;
bs.reset(new ByteStream());
bs->append((uint8_t*)&ism, sizeof(ism));
bs->append((uint8_t*)&ph, sizeof(ph));
*bs << error;
if (logIt)
{
@ -1982,7 +1986,7 @@ void BatchPrimitiveProcessor::writeErrorMsg(const string& error, uint16_t errCod
}
}
void BatchPrimitiveProcessor::writeProjectionPreamble()
void BatchPrimitiveProcessor::writeProjectionPreamble(SBS& bs)
{
ISMPacketHeader ism;
PrimitiveHeader ph;
@ -1997,36 +2001,36 @@ void BatchPrimitiveProcessor::writeProjectionPreamble()
ph.StepID = stepID;
ph.UniqueID = uniqueID;
serialized.reset(new ByteStream());
serialized->append((uint8_t*)&ism, sizeof(ism));
serialized->append((uint8_t*)&ph, sizeof(ph));
bs.reset(new ByteStream());
bs->append((uint8_t*)&ism, sizeof(ism));
bs->append((uint8_t*)&ph, sizeof(ph));
/* add-ons */
if (hasScan)
{
if (validCPData)
{
*serialized << (uint8_t)1;
*serialized << lbidForCP;
*serialized << ((uint8_t)cpDataFromDictScan);
*bs << (uint8_t)1;
*bs << lbidForCP;
*bs << ((uint8_t)cpDataFromDictScan);
if (UNLIKELY(hasWideColumnOut))
{
// PSA width
*serialized << (uint8_t)wideColumnWidthOut;
*serialized << min128Val;
*serialized << max128Val;
*bs << (uint8_t)wideColumnWidthOut;
*bs << min128Val;
*bs << max128Val;
}
else
{
*serialized << (uint8_t)utils::MAXLEGACYWIDTH; // width of min/max value
*serialized << (uint64_t)minVal;
*serialized << (uint64_t)maxVal;
*bs << (uint8_t)utils::MAXLEGACYWIDTH; // width of min/max value
*bs << (uint64_t)minVal;
*bs << (uint64_t)maxVal;
}
}
else
{
*serialized << (uint8_t)0;
*serialized << lbidForCP;
*bs << (uint8_t)0;
*bs << lbidForCP;
}
}
@ -2035,34 +2039,34 @@ void BatchPrimitiveProcessor::writeProjectionPreamble()
if (ot != ROW_GROUP)
{
*serialized << ridCount;
*bs << ridCount;
if (sendRidsAtDelivery)
{
*serialized << baseRid;
serialized->append((uint8_t*)relRids, ridCount << 1);
*bs << baseRid;
bs->append((uint8_t*)relRids, ridCount << 1);
}
}
}
void BatchPrimitiveProcessor::serializeElementTypes()
void BatchPrimitiveProcessor::serializeElementTypes(messageqcpp::SBS& bs)
{
*serialized << baseRid;
*serialized << ridCount;
serialized->append((uint8_t*)relRids, ridCount << 1);
serialized->append((uint8_t*)values, ridCount << 3);
*bs << baseRid;
*bs << ridCount;
bs->append((uint8_t*)relRids, ridCount << 1);
bs->append((uint8_t*)values, ridCount << 3);
}
void BatchPrimitiveProcessor::serializeStrings()
void BatchPrimitiveProcessor::serializeStrings(messageqcpp::SBS& bs)
{
*serialized << ridCount;
serialized->append((uint8_t*)absRids.get(), ridCount << 3);
*bs << ridCount;
bs->append((uint8_t*)absRids.get(), ridCount << 3);
for (uint32_t i = 0; i < ridCount; ++i)
*serialized << strValues[i];
*bs << strValues[i];
}
void BatchPrimitiveProcessor::sendResponse()
void BatchPrimitiveProcessor::sendResponse(messageqcpp::SBS& bs)
{
// Here is the fast path for local EM to PM interaction. PM puts into the
// input EM DEC queue directly.
@ -2073,12 +2077,12 @@ void BatchPrimitiveProcessor::sendResponse()
// is limited.
if (sendThread->flowControlEnabled())
{
sendThread->sendResult({serialized, sock, writelock, 0}, false);
sendThread->sendResult({bs, sock, writelock, 0}, false);
}
else
{
sock->write(serialized);
serialized.reset();
sock->write(bs);
bs.reset();
}
return;
@ -2088,20 +2092,20 @@ void BatchPrimitiveProcessor::sendResponse()
{
// newConnection should be set only for the first result of a batch job
// it tells sendthread it should consider it for the connection array
sendThread->sendResult(BPPSendThread::Msg_t(serialized, sock, writelock, sockIndex), newConnection);
sendThread->sendResult(BPPSendThread::Msg_t(bs, sock, writelock, sockIndex), newConnection);
newConnection = false;
}
else
{
boost::mutex::scoped_lock lk(*writelock);
sock->write(*serialized);
sock->write(*bs);
}
serialized.reset();
bs.reset();
}
/* The output of a filter chain is either ELEMENT_TYPE or STRING_ELEMENT_TYPE */
void BatchPrimitiveProcessor::makeResponse()
void BatchPrimitiveProcessor::makeResponse(messageqcpp::SBS& bs)
{
ISMPacketHeader ism;
PrimitiveHeader ph;
@ -2116,39 +2120,39 @@ void BatchPrimitiveProcessor::makeResponse()
ph.StepID = stepID;
ph.UniqueID = uniqueID;
serialized.reset(new ByteStream());
serialized->append((uint8_t*)&ism, sizeof(ism));
serialized->append((uint8_t*)&ph, sizeof(ph));
bs.reset(new ByteStream());
bs->append((uint8_t*)&ism, sizeof(ism));
bs->append((uint8_t*)&ph, sizeof(ph));
/* add-ons */
if (hasScan)
{
if (validCPData)
{
*serialized << (uint8_t)1;
*serialized << lbidForCP;
*serialized << ((uint8_t)cpDataFromDictScan);
*bs << (uint8_t)1;
*bs << lbidForCP;
*bs << ((uint8_t)cpDataFromDictScan);
if (UNLIKELY(hasWideColumnOut))
{
// PSA width
// Remove the assert for >16 bytes DTs.
assert(wideColumnWidthOut == datatypes::MAXDECIMALWIDTH);
*serialized << (uint8_t)wideColumnWidthOut;
*serialized << min128Val;
*serialized << max128Val;
*bs << (uint8_t)wideColumnWidthOut;
*bs << min128Val;
*bs << max128Val;
}
else
{
*serialized << (uint8_t)utils::MAXLEGACYWIDTH; // width of min/max value
*serialized << (uint64_t)minVal;
*serialized << (uint64_t)maxVal;
*bs << (uint8_t)utils::MAXLEGACYWIDTH; // width of min/max value
*bs << (uint64_t)minVal;
*bs << (uint64_t)maxVal;
}
}
else
{
*serialized << (uint8_t)0;
*serialized << lbidForCP;
*bs << (uint8_t)0;
*bs << lbidForCP;
}
}
@ -2156,9 +2160,9 @@ void BatchPrimitiveProcessor::makeResponse()
/* Take the rid and value arrays, munge into OutputType ot */
switch (ot)
{
case BPS_ELEMENT_TYPE: serializeElementTypes(); break;
case BPS_ELEMENT_TYPE: serializeElementTypes(bs); break;
case STRING_ELEMENT_TYPE: serializeStrings(); break;
case STRING_ELEMENT_TYPE: serializeStrings(bs); break;
default:
{
@ -2166,15 +2170,13 @@ void BatchPrimitiveProcessor::makeResponse()
oss << "BPP: makeResponse(): Bad output type: " << ot;
throw logic_error(oss.str());
}
// throw logic_error("BPP: makeResponse(): Bad output type");
}
*serialized << cachedIO;
*bs << cachedIO;
cachedIO = 0;
*serialized << physIO;
*bs << physIO;
physIO = 0;
*serialized << touchedBlocks;
*bs << touchedBlocks;
touchedBlocks = 0;
// cout << "sent physIO=" << physIO << " cachedIO=" << cachedIO <<
@ -2230,20 +2232,24 @@ int BatchPrimitiveProcessor::operator()()
validCPData = false;
cpDataFromDictScan = false;
auto alloc = exemgr::globServiceExeMgr->getRm().getAllocator<messageqcpp::BSBufType>();
messageqcpp::SBS bs(new ByteStream(&alloc));
#ifdef PRIMPROC_STOPWATCH
stopwatch->start("BPP() execute");
execute(stopwatch);
stopwatch->stop("BPP() execute");
#else
execute();
execute(bs);
#endif
if (projectCount == 0 && ot != ROW_GROUP)
makeResponse();
makeResponse(bs);
try
{
sendResponse();
sendResponse(bs);
}
catch (std::exception& e)
{
@ -2717,7 +2723,7 @@ inline void BatchPrimitiveProcessor::getJoinResults(const Row& r, uint32_t jInde
{
bool hasNullValue = false;
for (unsigned int column: tlLargeSideKeyColumns[jIndex])
for (unsigned int column : tlLargeSideKeyColumns[jIndex])
{
if (r.isNullValue(column))
{

44
primitives/primproc/batchprimitiveprocessor.h
View File

@ -33,9 +33,10 @@
#include <boost/scoped_array.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/scoped_ptr.hpp>
#include <tr1/unordered_map>
#include <unordered_map>
#include <boost/thread.hpp>
#include "countingallocator.h"
#include "errorcodes.h"
#include "serializeable.h"
#include "messagequeue.h"
@ -189,20 +190,20 @@ class BatchPrimitiveProcessor
#ifdef PRIMPROC_STOPWATCH
void execute(logging::StopWatch* stopwatch);
#else
void execute();
void execute(messageqcpp::SBS& bs);
#endif
void writeProjectionPreamble();
void makeResponse();
void sendResponse();
void writeProjectionPreamble(messageqcpp::SBS& bs);
void makeResponse(messageqcpp::SBS& bs);
void sendResponse(messageqcpp::SBS& bs);
/* Used by scan operations to increment the LBIDs in successive steps */
void nextLBID();
/* these send relative rids, should this be abs rids? */
void serializeElementTypes();
void serializeStrings();
void serializeElementTypes(messageqcpp::SBS& bs);
void serializeStrings(messageqcpp::SBS& bs);
void asyncLoadProjectColumns();
void writeErrorMsg(const std::string& error, uint16_t errCode, bool logIt = true, bool critical = true);
void writeErrorMsg(messageqcpp::SBS& bs, const std::string& error, uint16_t errCode, bool logIt = true, bool critical = true);
BPSOutputType ot;
@ -269,7 +270,7 @@ class BatchPrimitiveProcessor
uint32_t physIO, cachedIO, touchedBlocks;
SP_UM_IOSOCK sock;
messageqcpp::SBS serialized;
// messageqcpp::SBS serialized;
SP_UM_MUTEX writelock;
// MCOL-744 using pthread mutex instead of Boost mutex because
@ -308,16 +309,23 @@ class BatchPrimitiveProcessor
bool hasRowGroup;
/* Rowgroups + join */
typedef std::tr1::unordered_multimap<uint64_t, uint32_t, joiner::TupleJoiner::hasher,
std::equal_to<uint64_t>,
utils::STLPoolAllocator<std::pair<const uint64_t, uint32_t>>>
TJoiner;
// typedef std::unordered_multimap<uint64_t, uint32_t, joiner::TupleJoiner::hasher,
// std::equal_to<uint64_t>,
// utils::STLPoolAllocator<std::pair<const uint64_t, uint32_t>>>
// TJoiner;
using TJoiner =
std::unordered_multimap<uint64_t, uint32_t, joiner::TupleJoiner::hasher, std::equal_to<uint64_t>,
allocators::CountingAllocator<std::pair<const uint64_t, uint32_t>>>;
typedef std::tr1::unordered_multimap<
joiner::TypelessData, uint32_t, joiner::TupleJoiner::TypelessDataHasher,
joiner::TupleJoiner::TypelessDataComparator,
utils::STLPoolAllocator<std::pair<const joiner::TypelessData, uint32_t>>>
TLJoiner;
// typedef std::unordered_multimap<
// joiner::TypelessData, uint32_t, joiner::TupleJoiner::TypelessDataHasher,
// joiner::TupleJoiner::TypelessDataComparator,
// utils::STLPoolAllocator<std::pair<const joiner::TypelessData, uint32_t>>>
// TLJoiner;
using TLJoiner =
std::unordered_multimap<joiner::TypelessData, uint32_t, joiner::TupleJoiner::TypelessDataHasher,
joiner::TupleJoiner::TypelessDataComparator,
allocators::CountingAllocator<std::pair<const joiner::TypelessData, uint32_t>>>;
bool generateJoinedRowGroup(rowgroup::Row& baseRow, const uint32_t depth = 0);
/* generateJoinedRowGroup helper fcns & vars */

12
primitives/primproc/columncommand.cpp
View File

@ -654,7 +654,7 @@ void ColumnCommand::fillInPrimitiveMessageHeader(const int8_t outputType, const
}
/* Assumes OT_DATAVALUE */
void ColumnCommand::projectResult()
void ColumnCommand::projectResult(messageqcpp::SBS& bs)
{
auto nvals = outMsg->NVALS;
if (primMsg->NVALS != nvals || nvals != bpp->ridCount)
@ -687,8 +687,8 @@ void ColumnCommand::projectResult()
idbassert(primMsg->NVALS == nvals);
idbassert(bpp->ridCount == nvals);
uint32_t valuesByteSize = nvals * colType.colWidth;
*bpp->serialized << valuesByteSize;
bpp->serialized->append(primitives::getFirstValueArrayPosition(outMsg), valuesByteSize);
*bs << valuesByteSize;
bs->append(primitives::getFirstValueArrayPosition(outMsg), valuesByteSize);
}
void ColumnCommand::removeRowsFromRowGroup(RowGroup& rg)
@ -815,19 +815,19 @@ void ColumnCommand::projectResultRG(RowGroup& rg, uint32_t pos)
}
}
void ColumnCommand::project()
void ColumnCommand::project(messageqcpp::SBS& bs)
{
/* bpp->ridCount == 0 would signify a scan operation */
if (bpp->ridCount == 0)
{
*bpp->serialized << (uint32_t)0;
*bs << (uint32_t)0;
blockCount += colType.colWidth;
return;
}
makeStepMsg();
issuePrimitive();
projectResult();
projectResult(bs);
}
void ColumnCommand::projectIntoRowGroup(RowGroup& rg, uint32_t pos)

5
primitives/primproc/columncommand.h
View File

@ -31,6 +31,7 @@
#pragma once
#include <memory>
#include "bytestream.h"
#include "columnwidth.h"
#include "command.h"
#include "calpontsystemcatalog.h"
@ -71,7 +72,7 @@ class ColumnCommand : public Command
void execute() override;
void execute(int64_t* vals); // used by RTSCommand to redirect values
void prep(int8_t outputType, bool absRids) override;
void project() override;
void project(messageqcpp::SBS& bs) override;
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t pos) override;
void nextLBID() override;
bool isScan()
@ -151,7 +152,7 @@ class ColumnCommand : public Command
template <int W>
void _process_OT_DATAVALUE();
void process_OT_ROWGROUP();
void projectResult();
void projectResult(messageqcpp::SBS& bs);
template <typename T>
void _projectResultRGLoop(rowgroup::Row& r, const T* valuesArray, const uint32_t offset);
template <int W>

2
primitives/primproc/command.h
View File

@ -54,7 +54,7 @@ class Command
virtual ~Command();
virtual void execute() = 0;
virtual void project() = 0;
virtual void project(messageqcpp::SBS& bs) = 0;
virtual void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t columnPosition) = 0;
virtual uint64_t getLBID() = 0;
virtual void getLBIDList(uint32_t loopCount, std::vector<int64_t>* out)

14
primitives/primproc/dictstep.cpp
View File

@ -414,7 +414,7 @@ void DictStep::_execute()
}
/* This will do the same thing as execute() but put the result in bpp->serialized */
void DictStep::_project()
void DictStep::_project(messageqcpp::SBS& bs)
{
/* Need to loop over bpp->values, issuing a primitive for each LBID */
uint32_t i;
@ -466,13 +466,13 @@ void DictStep::_project()
}
idbassert(tmpResultCounter == bpp->ridCount);
*bpp->serialized << totalResultLength;
*bs << totalResultLength;
// cout << "_project() total length = " << totalResultLength << endl;
for (i = 0; i < tmpResultCounter; i++)
{
// cout << "serializing " << tmpStrings[i] << endl;
*bpp->serialized << tmpStrings[i];
*bs << tmpStrings[i];
}
// cout << "DS: /_project() l: " << l_lbid << endl;
@ -645,16 +645,16 @@ void DictStep::_projectToRG(RowGroup& rg, uint32_t col)
// << endl;
}
void DictStep::project()
void DictStep::project(messageqcpp::SBS& bs)
{
values = bpp->values;
_project();
_project(bs);
}
void DictStep::project(int64_t* vals)
void DictStep::project(messageqcpp::SBS& bs, int64_t* vals)
{
values = vals;
_project();
_project(bs);
}
void DictStep::projectIntoRowGroup(RowGroup& rg, uint32_t col)

6
primitives/primproc/dictstep.h
View File

@ -42,8 +42,8 @@ class DictStep : public Command
~DictStep() override;
void execute() override;
void project() override;
void project(int64_t* vals); // used by RTSCommand to redirect input
void project(messageqcpp::SBS& bs) override;
void project(messageqcpp::SBS& bs, int64_t* vals); // used by RTSCommand to redirect input
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t row) override;
void projectIntoRowGroup(rowgroup::RowGroup& rg, int64_t* vals, uint32_t col);
uint64_t getLBID() override;
@ -95,7 +95,7 @@ class DictStep : public Command
void processResult();
void projectResult(std::string* tmpStrings);
void projectResult(StringPtr* tmpStrings);
void _project();
void _project(messageqcpp::SBS& bs);
void _projectToRG(rowgroup::RowGroup& rg, uint32_t col);
// struct used for scratch space

2
primitives/primproc/filtercommand.cpp
View File

@ -201,7 +201,7 @@ void FilterCommand::prep(int8_t outputType, bool absRids)
{
}
void FilterCommand::project()
void FilterCommand::project(messageqcpp::SBS& bs)
{
}

2
primitives/primproc/filtercommand.h
View File

@ -45,7 +45,7 @@ class FilterCommand : public Command
// virtuals from base class -- Command
void execute() override;
void project() override;
void project(messageqcpp::SBS& bs) override;
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col) override;
uint64_t getLBID() override;
void nextLBID() override;

14
primitives/primproc/passthrucommand.cpp
View File

@ -60,11 +60,11 @@ void PassThruCommand::execute()
// throw logic_error("PassThruCommand isn't a filter step");
}
void PassThruCommand::project()
void PassThruCommand::project(messageqcpp::SBS& bs)
{
uint32_t i;
*bpp->serialized << (uint32_t)(bpp->ridCount * colWidth);
*bs << (uint32_t)(bpp->ridCount * colWidth);
#if 0
cout << "pass thru serializing " << (uint32_t) (bpp->ridCount * colWidth) << " bytes:\n";
cout << "at relative position " << bpp->serialized->length() - sizeof(ISMPacketHeader) - sizeof(PrimitiveHeader) - 4 << endl;
@ -76,25 +76,25 @@ void PassThruCommand::project()
switch (colWidth)
{
case 16: bpp->serialized->append((uint8_t*)bpp->wide128Values, bpp->ridCount << 4); break;
case 16: bs->append((uint8_t*)bpp->wide128Values, bpp->ridCount << 4); break;
case 8: bpp->serialized->append((uint8_t*)bpp->values, bpp->ridCount << 3); break;
case 8: bs->append((uint8_t*)bpp->values, bpp->ridCount << 3); break;
case 4:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint32_t)bpp->values[i];
*bs << (uint32_t)bpp->values[i];
break;
case 2:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint16_t)bpp->values[i];
*bs << (uint16_t)bpp->values[i];
break;
case 1:
for (i = 0; i < bpp->ridCount; i++)
*bpp->serialized << (uint8_t)bpp->values[i];
*bs << (uint8_t)bpp->values[i];
break;

3
primitives/primproc/passthrucommand.h
View File

@ -30,6 +30,7 @@
#pragma once
#include "bytestream.h"
#include "command.h"
namespace primitiveprocessor
@ -42,7 +43,7 @@ class PassThruCommand : public Command
void prep(int8_t outputType, bool makeAbsRids) override;
void execute() override;
void project() override;
void project(messageqcpp::SBS& bs) override;
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col) override;
uint64_t getLBID() override;
void nextLBID() override;

8
primitives/primproc/primproc.cpp
View File

@ -121,7 +121,6 @@ void setupSignalHandlers()
sigset_t sigset;
sigemptyset(&sigset);
sigaddset(&sigset, SIGPIPE);
sigaddset(&sigset, SIGUSR1);
sigaddset(&sigset, SIGUSR2);
sigprocmask(SIG_BLOCK, &sigset, 0);
@ -337,12 +336,15 @@ int ServicePrimProc::Child()
return 2;
}
bool runningWithExeMgr = true;
auto* rm = joblist::ResourceManager::instance(runningWithExeMgr, cf);
utils::USpaceSpinLock startupRaceLock(getStartupRaceFlag());
std::thread exeMgrThread(
[this, cf]()
[this, rm]()
{
exemgr::Opt opt;
exemgr::globServiceExeMgr = new exemgr::ServiceExeMgr(opt, cf);
exemgr::globServiceExeMgr = new exemgr::ServiceExeMgr(opt, rm);
// primitive delay to avoid 'not connected to PM' log error messages
// from EM. PrimitiveServer::start() releases SpinLock after sockets
// are available.

7
primitives/primproc/rtscommand.cpp
View File

@ -31,6 +31,7 @@
#include <unistd.h>
#include "bpp.h"
#include "bytestream.h"
#include "exceptclasses.h"
using namespace std;
@ -53,7 +54,7 @@ void RTSCommand::execute()
throw logic_error("RTSCommand shouldn't be used for filter steps");
}
void RTSCommand::project()
void RTSCommand::project(messageqcpp::SBS& bs)
{
uint32_t i;
@ -70,7 +71,7 @@ void RTSCommand::project()
// need something in values
dict.project();
dict.project(bs);
}
else
{
@ -99,7 +100,7 @@ void RTSCommand::project()
}
}
dict.project(tmpValues);
dict.project(bs, tmpValues);
}
}

3
primitives/primproc/rtscommand.h
View File

@ -30,6 +30,7 @@
#pragma once
#include "bytestream.h"
#include "command.h"
#include <boost/scoped_ptr.hpp>
#include <memory>
@ -43,7 +44,7 @@ class RTSCommand : public Command
~RTSCommand() override;
void execute() override;
void project() override;
void project(messageqcpp::SBS& bs) override;
void projectIntoRowGroup(rowgroup::RowGroup& rg, uint32_t col) override;
uint64_t getLBID() override;
void nextLBID() override;

18
primitives/primproc/serviceexemgr.cpp
View File

@ -89,22 +89,8 @@ void startRssMon(size_t maxPct, int pauseSeconds);
void added_a_pm(int)
{
logging::LoggingID logid(21, 0, 0);
logging::Message::Args args1;
logging::Message msg(1);
args1.add("exeMgr caught SIGHUP. Resetting connections");
msg.format(args1);
std::cout << msg.msg().c_str() << std::endl;
logging::Logger logger(logid.fSubsysID);
logger.logMessage(logging::LOG_TYPE_DEBUG, msg, logid);
auto* dec = exemgr::globServiceExeMgr->getDec();
if (dec)
{
oam::OamCache* oamCache = oam::OamCache::makeOamCache();
oamCache->forceReload();
dec->Setup();
}
int64_t num = globServiceExeMgr->getRm().availableMemory();
std::cout << "Total UM memory available: " << num << std::endl;
}
void printTotalUmMemory(int sig)

80
primitives/primproc/serviceexemgr.h
View File

@ -94,6 +94,8 @@ class Opt
}
};
void printTotalUmMemory(int sig);
class ServiceExeMgr : public Service, public Opt
{
using SessionMemMap_t = std::map<uint32_t, size_t>;
@ -111,50 +113,40 @@ class ServiceExeMgr : public Service, public Opt
logger.logMessage(type, message, logid);
}
public:
ServiceExeMgr(const Opt& opt) : Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16))
{
bool runningWithExeMgr = true;
rm_ = joblist::ResourceManager::instance(runningWithExeMgr);
}
ServiceExeMgr(const Opt& opt, config::Config* aConfig)
: Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16))
{
bool runningWithExeMgr = true;
rm_ = joblist::ResourceManager::instance(runningWithExeMgr, aConfig);
}
void LogErrno() override
{
log(logging::LOG_TYPE_CRITICAL, std::string(strerror(errno)));
}
void ParentLogChildMessage(const std::string& str) override
{
log(logging::LOG_TYPE_INFO, str);
}
int Child() override;
int Run()
{
return m_fg ? Child() : RunForking();
}
static const constexpr unsigned logDefaultMsg = logging::M0000;
static const constexpr unsigned logDbProfStartStatement = logging::M0028;
static const constexpr unsigned logDbProfEndStatement = logging::M0029;
static const constexpr unsigned logStartSql = logging::M0041;
static const constexpr unsigned logEndSql = logging::M0042;
static const constexpr unsigned logRssTooBig = logging::M0044;
static const constexpr unsigned logDbProfQueryStats = logging::M0047;
static const constexpr unsigned logExeMgrExcpt = logging::M0055;
// If any flags other than the table mode flags are set, produce output to screeen
static const constexpr uint32_t flagsWantOutput =
(0xffffffff & ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH &
~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF);
logging::Logger& getLogger()
{
return msgLog_;
}
void updateSessionMap(const size_t pct)
{
std::lock_guard<std::mutex> lk(sessionMemMapMutex_);
public:
ServiceExeMgr(const Opt& opt, joblist::ResourceManager* rm) : Service("ExeMgr"), Opt(opt), msgLog_(logging::Logger(16)), rm_(rm)
{ }
void LogErrno() override
{
log(logging::LOG_TYPE_CRITICAL, std::string(strerror(errno)));
}
void ParentLogChildMessage(const std::string& str) override
{
log(logging::LOG_TYPE_INFO, str);
}
int Child() override;
int Run()
{
return m_fg ? Child() : RunForking();
}
static const constexpr unsigned logDefaultMsg = logging::M0000;
static const constexpr unsigned logDbProfStartStatement = logging::M0028;
static const constexpr unsigned logDbProfEndStatement = logging::M0029;
static const constexpr unsigned logStartSql = logging::M0041;
static const constexpr unsigned logEndSql = logging::M0042;
static const constexpr unsigned logRssTooBig = logging::M0044;
static const constexpr unsigned logDbProfQueryStats = logging::M0047;
static const constexpr unsigned logExeMgrExcpt = logging::M0055;
// If any flags other than the table mode flags are set, produce output to screeen
static const constexpr uint32_t flagsWantOutput = (0xffffffff & ~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_AUTOSWITCH &
~execplan::CalpontSelectExecutionPlan::TRACE_TUPLE_OFF);
logging::Logger& getLogger()
{
return msgLog_;
}
void updateSessionMap(const size_t pct)
{
std::lock_guard<std::mutex> lk(sessionMemMapMutex_);
for (auto mapIter = sessionMemMap_.begin(); mapIter != sessionMemMap_.end(); ++mapIter)
{