database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-12-10 22:42:30 +03:00
Code Activity
Files
c97e58c9b51aab2667b5c98d4d810ff76d9dfafa
mariadb-columnstore-engine/dbcon/joblist/tupleunion.cpp
Patrick LeBlanc a09a9d5d0f Mass substitution 'Corporaton' -> 'Corporation'
2019-08-07 14:43:25 -05:00

1424 lines
48 KiB
C++
Raw Blame History

/* Copyright (C) 2014 InfiniDB, Inc.
Copyright (C) 2019 MariaDB Corporation
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
/*****************************************************************************
* $Id: tupleunion.cpp 9665 2013-07-02 21:47:39Z pleblanc $
*
****************************************************************************/
#include <string>
#include <boost/thread.hpp>
#include <boost/thread/mutex.hpp>
#include <boost/uuid/uuid_io.hpp>
#include "querytele.h"
using namespace querytele;
#include "dataconvert.h"
#include "hasher.h"
#include "jlf_common.h"
#include "resourcemanager.h"
#include "tupleunion.h"
using namespace std;
using namespace std::tr1;
using namespace boost;
using namespace execplan;
using namespace rowgroup;
using namespace dataconvert;
#ifndef __linux__
#ifndef M_LN10
#define M_LN10 2.30258509299404568402 /* log_e 10 */
#endif
namespace
{
//returns the value of 10 raised to the power x.
inline double exp10(double x)
{
return exp(x * M_LN10);
}
}
#endif
namespace joblist
{
inline uint64_t TupleUnion::Hasher::operator()(const RowPosition& p) const
{
Row& row = ts->row;
if (p.group & RowPosition::normalizedFlag)
ts->normalizedData[p.group & ~RowPosition::normalizedFlag].getRow(p.row, &row);
else
ts->rowMemory[p.group].getRow(p.row, &row);
return row.hash();
}
inline bool TupleUnion::Eq::operator()(const RowPosition& d1, const RowPosition& d2) const
{
Row& r1 = ts->row, &r2 = ts->row2;
if (d1.group & RowPosition::normalizedFlag)
ts->normalizedData[d1.group & ~RowPosition::normalizedFlag].getRow(d1.row, &r1);
else
ts->rowMemory[d1.group].getRow(d1.row, &r1);
if (d2.group & RowPosition::normalizedFlag)
ts->normalizedData[d2.group & ~RowPosition::normalizedFlag].getRow(d2.row, &r2);
else
ts->rowMemory[d2.group].getRow(d2.row, &r2);
return r1.equals(r2);
}
TupleUnion::TupleUnion(CalpontSystemCatalog::OID tableOID, const JobInfo& jobInfo) :
JobStep(jobInfo),
fTableOID(tableOID),
output(NULL),
outputIt(-1),
memUsage(0),
rm(jobInfo.rm),
allocator(64 * 1024 * 1024 + 1),
runnersDone(0),
distinctCount(0),
distinctDone(0),
fRowsReturned(0),
runRan(false),
joinRan(false),
sessionMemLimit(jobInfo.umMemLimit),
fTimeZone(jobInfo.timeZone)
{
uniquer.reset(new Uniquer_t(10, Hasher(this), Eq(this), allocator));
fExtendedInfo = "TUN: ";
fQtc.stepParms().stepType = StepTeleStats::T_TUN;
}
TupleUnion::~TupleUnion()
{
rm->returnMemory(memUsage, sessionMemLimit);
if (!runRan && output)
output->endOfInput();
}
CalpontSystemCatalog::OID TupleUnion::tableOid() const
{
return fTableOID;
}
void TupleUnion::setInputRowGroups(const vector<rowgroup::RowGroup>& in)
{
inputRGs = in;
}
void TupleUnion::setOutputRowGroup(const rowgroup::RowGroup& out)
{
outputRG = out;
rowLength = outputRG.getRowSizeWithStrings();
}
void TupleUnion::setDistinctFlags(const vector<bool>& v)
{
distinctFlags = v;
}
void TupleUnion::readInput(uint32_t which)
{
/* The handling of the output got a little kludgey with the string table enhancement.
* When there is no distinct check, the outputs are all generated independently of
* each other locally in this fcn. When there is a distinct check, threads
* share the output, which is built in the 'rowMemory' vector rather than in
* thread-local memory. Building the result in a common space allows us to
* store 8-byte offsets in rowMemory rather than 16-bytes for absolute pointers.
*/
RowGroupDL* dl = NULL;
bool more = true;
RGData inRGData, outRGData, *tmpRGData;
uint32_t it = numeric_limits<uint32_t>::max();
RowGroup l_inputRG, l_outputRG, l_tmpRG;
Row inRow, outRow, tmpRow;
bool distinct;
uint64_t memUsageBefore, memUsageAfter, memDiff;
StepTeleStats sts;
sts.query_uuid = fQueryUuid;
sts.step_uuid = fStepUuid;
l_outputRG = outputRG;
dl = inputs[which];
l_inputRG = inputRGs[which];
l_inputRG.initRow(&inRow);
l_outputRG.initRow(&outRow);
distinct = distinctFlags[which];
if (distinct)
{
l_tmpRG = outputRG;
tmpRGData = &normalizedData[which];
l_tmpRG.initRow(&tmpRow);
l_tmpRG.setData(tmpRGData);
l_tmpRG.resetRowGroup(0);
l_tmpRG.getRow(0, &tmpRow);
}
else
{
outRGData = RGData(l_outputRG);
l_outputRG.setData(&outRGData);
l_outputRG.resetRowGroup(0);
l_outputRG.getRow(0, &outRow);
}
try
{
it = dl->getIterator();
more = dl->next(it, &inRGData);
if (dlTimes.FirstReadTime().tv_sec == 0)
dlTimes.setFirstReadTime();
if (fStartTime == -1)
{
sts.msg_type = StepTeleStats::ST_START;
sts.total_units_of_work = 1;
postStepStartTele(sts);
}
while (more && !cancelled())
{
/*
normalize each row
if distinct flag is set
copy the row into the output and test for uniqueness
if unique, increment the row count
else
copy the row into the output & inc row count
*/
l_inputRG.setData(&inRGData);
l_inputRG.getRow(0, &inRow);
if (distinct)
{
memDiff = 0;
l_tmpRG.resetRowGroup(0);
l_tmpRG.getRow(0, &tmpRow);
l_tmpRG.setRowCount(l_inputRG.getRowCount());
for (uint32_t i = 0; i < l_inputRG.getRowCount(); i++, inRow.nextRow(),
tmpRow.nextRow())
normalize(inRow, &tmpRow);
l_tmpRG.getRow(0, &tmpRow);
{
mutex::scoped_lock lk(uniquerMutex);
getOutput(&l_outputRG, &outRow, &outRGData);
memUsageBefore = allocator.getMemUsage();
for (uint32_t i = 0; i < l_tmpRG.getRowCount(); i++, tmpRow.nextRow())
{
pair<Uniquer_t::iterator, bool> inserted;
inserted = uniquer->insert(RowPosition(which | RowPosition::normalizedFlag, i));
if (inserted.second)
{
copyRow(tmpRow, &outRow);
const_cast<RowPosition&>(*(inserted.first)) = RowPosition(rowMemory.size() - 1, l_outputRG.getRowCount());
memDiff += outRow.getRealSize();
addToOutput(&outRow, &l_outputRG, true, outRGData);
}
}
memUsageAfter = allocator.getMemUsage();
memDiff += (memUsageAfter - memUsageBefore);
memUsage += memDiff;
}
if (!rm->getMemory(memDiff, sessionMemLimit))
{
fLogger->logMessage(logging::LOG_TYPE_INFO, logging::ERR_UNION_TOO_BIG);
if (status() == 0) // preserve existing error code
{
errorMessage(logging::IDBErrorInfo::instance()->errorMsg(
logging::ERR_UNION_TOO_BIG));
status(logging::ERR_UNION_TOO_BIG);
}
abort();
}
}
else
{
for (uint32_t i = 0; i < l_inputRG.getRowCount(); i++, inRow.nextRow())
{
normalize(inRow, &outRow);
addToOutput(&outRow, &l_outputRG, false, outRGData);
}
}
more = dl->next(it, &inRGData);
}
}
catch (...)
{
if (status() == 0)
{
errorMessage("Union step caught an unknown exception.");
status(logging::unionStepErr);
fLogger->logMessage(logging::LOG_TYPE_CRITICAL, "Union step caught an unknown exception.");
}
abort();
}
/* make sure that the input was drained before exiting. This can happen if the
query was aborted */
if (dl && it != numeric_limits<uint32_t>::max())
while (more)
more = dl->next(it, &inRGData);
{
mutex::scoped_lock lock1(uniquerMutex);
mutex::scoped_lock lock2(sMutex);
if (!distinct && l_outputRG.getRowCount() > 0)
output->insert(outRGData);
if (distinct)
{
getOutput(&l_outputRG, &outRow, &outRGData);
if (++distinctDone == distinctCount && l_outputRG.getRowCount() > 0)
output->insert(outRGData);
}
if (++runnersDone == fInputJobStepAssociation.outSize())
{
output->endOfInput();
sts.msg_type = StepTeleStats::ST_SUMMARY;
sts.total_units_of_work = sts.units_of_work_completed = 1;
sts.rows = fRowsReturned;
postStepSummaryTele(sts);
if (traceOn())
{
dlTimes.setLastReadTime();
dlTimes.setEndOfInputTime();
time_t t = time (0);
char timeString[50];
ctime_r (&t, timeString);
timeString[strlen (timeString ) - 1] = '\0';
ostringstream logStr;
logStr << "ses:" << fSessionId << " st: " << fStepId << " finished at "
<< timeString << "; total rows returned-" << fRowsReturned << endl
<< "\t1st read " << dlTimes.FirstReadTimeString()
<< "; EOI " << dlTimes.EndOfInputTimeString() << "; runtime-"
<< JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime())
<< "s;\n\tUUID " << uuids::to_string(fStepUuid) << endl
<< "\tJob completion status " << status() << endl;
logEnd(logStr.str().c_str());
fExtendedInfo += logStr.str();
formatMiniStats();
}
}
}
}
uint32_t TupleUnion::nextBand(messageqcpp::ByteStream& bs)
{
RGData mem;
bool more;
uint32_t ret = 0;
bs.restart();
more = output->next(outputIt, &mem);
if (more)
outputRG.setData(&mem);
else
{
mem = RGData(outputRG, 0);
outputRG.setData(&mem);
outputRG.resetRowGroup(0);
outputRG.setStatus(status());
}
outputRG.serializeRGData(bs);
ret = outputRG.getRowCount();
return ret;
}
void TupleUnion::getOutput(RowGroup* rg, Row* row, RGData* data)
{
if (UNLIKELY(rowMemory.empty()))
{
*data = RGData(*rg);
rg->setData(data);
rg->resetRowGroup(0);
rowMemory.push_back(*data);
}
else
{
*data = rowMemory.back();
rg->setData(data);
}
rg->getRow(rg->getRowCount(), row);
}
void TupleUnion::addToOutput(Row* r, RowGroup* rg, bool keepit,
RGData& data)
{
r->nextRow();
rg->incRowCount();
fRowsReturned++;
if (rg->getRowCount() == 8192)
{
{
mutex::scoped_lock lock(sMutex);
output->insert(data);
}
data = RGData(*rg);
rg->setData(&data);
rg->resetRowGroup(0);
rg->getRow(0, r);
if (keepit)
rowMemory.push_back(data);
}
}
void TupleUnion::normalize(const Row& in, Row* out)
{
uint32_t i;
out->setRid(0);
for (i = 0; i < out->getColumnCount(); i++)
{
if (in.isNullValue(i))
{
writeNull(out, i);
continue;
}
switch (in.getColTypes()[i])
{
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
case CalpontSystemCatalog::BIGINT:
switch (out->getColTypes()[i])
{
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
case CalpontSystemCatalog::BIGINT:
if (out->getScale(i) || in.getScale(i))
goto dec1;
out->setIntField(in.getIntField(i), i);
break;
case CalpontSystemCatalog::UTINYINT:
case CalpontSystemCatalog::USMALLINT:
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
case CalpontSystemCatalog::UBIGINT:
if (in.getScale(i))
goto dec1;
out->setUintField(in.getUintField(i), i);
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
{
ostringstream os;
if (in.getScale(i))
{
double d = in.getIntField(i);
d /= exp10(in.getScale(i));
os.precision(15);
os << d;
}
else
os << in.getIntField(i);
out->setStringField(os.str(), i);
break;
}
case CalpontSystemCatalog::DATE:
case CalpontSystemCatalog::DATETIME:
case CalpontSystemCatalog::TIME:
case CalpontSystemCatalog::TIMESTAMP:
throw logic_error("TupleUnion::normalize(): tried to normalize an int to a timestamp, time, date or datetime");
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
{
int scale = in.getScale(i);
if (scale != 0)
{
float f = in.getIntField(i);
f /= (uint64_t) pow(10.0, scale);
out->setFloatField(f, i);
}
else
out->setFloatField(in.getIntField(i), i);
break;
}
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
{
int scale = in.getScale(i);
if (scale != 0)
{
double d = in.getIntField(i);
d /= (uint64_t) pow(10.0, scale);
out->setDoubleField(d, i);
}
else
out->setDoubleField(in.getIntField(i), i);
break;
}
case CalpontSystemCatalog::LONGDOUBLE:
{
int scale = in.getScale(i);
long double d = in.getIntField(i);
if (scale != 0)
{
d /= (uint64_t) pow(10.0, scale);
}
out->setLongDoubleField(d, i);
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
dec1:
uint64_t val = in.getIntField(i);
int diff = out->getScale(i) - in.getScale(i);
if (diff < 0)
val /= (uint64_t) pow((double) 10, (double) - diff);
else
val *= (uint64_t) pow((double) 10, (double) diff);
out->setIntField(val, i);
break;
}
default:
ostringstream os;
os << "TupleUnion::normalize(): tried an illegal conversion: integer to "
<< out->getColTypes()[i];
throw logic_error(os.str());
}
break;
case CalpontSystemCatalog::UTINYINT:
case CalpontSystemCatalog::USMALLINT:
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
case CalpontSystemCatalog::UBIGINT:
switch (out->getColTypes()[i])
{
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
case CalpontSystemCatalog::BIGINT:
if (out->getScale(i))
goto dec2;
out->setIntField(in.getUintField(i), i);
break;
case CalpontSystemCatalog::UTINYINT:
case CalpontSystemCatalog::USMALLINT:
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
case CalpontSystemCatalog::UBIGINT:
out->setUintField(in.getUintField(i), i);
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
{
ostringstream os;
if (in.getScale(i))
{
double d = in.getUintField(i);
d /= exp10(in.getScale(i));
os.precision(15);
os << d;
}
else
os << in.getUintField(i);
out->setStringField(os.str(), i);
break;
}
case CalpontSystemCatalog::DATE:
case CalpontSystemCatalog::DATETIME:
case CalpontSystemCatalog::TIME:
case CalpontSystemCatalog::TIMESTAMP:
throw logic_error("TupleUnion::normalize(): tried to normalize an int to a timestamp, time, date or datetime");
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
{
int scale = in.getScale(i);
if (scale != 0)
{
float f = in.getUintField(i);
f /= (uint64_t) pow(10.0, scale);
out->setFloatField(f, i);
}
else
out->setFloatField(in.getUintField(i), i);
break;
}
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
{
int scale = in.getScale(i);
if (scale != 0)
{
double d = in.getUintField(i);
d /= (uint64_t) pow(10.0, scale);
out->setDoubleField(d, i);
}
else
out->setDoubleField(in.getUintField(i), i);
break;
}
case CalpontSystemCatalog::LONGDOUBLE:
{
int scale = in.getScale(i);
if (scale != 0)
{
long double d = in.getUintField(i);
d /= (uint64_t) pow(10.0, scale);
out->setLongDoubleField(d, i);
}
else
out->setLongDoubleField(in.getUintField(i), i);
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
dec2:
uint64_t val = in.getIntField(i);
int diff = out->getScale(i) - in.getScale(i);
if (diff < 0)
val /= (uint64_t) pow((double) 10, (double) - diff);
else
val *= (uint64_t) pow((double) 10, (double) diff);
out->setIntField(val, i);
break;
}
default:
ostringstream os;
os << "TupleUnion::normalize(): tried an illegal conversion: integer to "
<< out->getColTypes()[i];
throw logic_error(os.str());
}
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
switch (out->getColTypes()[i])
{
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
out->setStringField(in.getStringField(i), i);
break;
default:
{
ostringstream os;
os << "TupleUnion::normalize(): tried an illegal conversion: string to "
<< out->getColTypes()[i];
throw logic_error(os.str());
}
}
break;
case CalpontSystemCatalog::DATE:
switch (out->getColTypes()[i])
{
case CalpontSystemCatalog::DATE:
out->setIntField(in.getIntField(i), i);
break;
case CalpontSystemCatalog::DATETIME:
{
uint64_t date = in.getUintField(i);
date &= ~0x3f; // zero the 'spare' field
date <<= 32;
out->setUintField(date, i);
break;
}
case CalpontSystemCatalog::TIMESTAMP:
{
dataconvert::Date date(in.getUintField(i));
dataconvert::MySQLTime m_time;
m_time.year = date.year;
m_time.month = date.month;
m_time.day = date.day;
m_time.hour = 0;
m_time.minute = 0;
m_time.second = 0;
m_time.second_part = 0;
dataconvert::TimeStamp timeStamp;
bool isValid = true;
int64_t seconds = dataconvert::mySQLTimeToGmtSec(m_time, fTimeZone, isValid);
if (!isValid)
{
timeStamp.reset();
}
else
{
timeStamp.second = seconds;
timeStamp.msecond = m_time.second_part;
}
uint64_t outValue = (uint64_t) *(reinterpret_cast<uint64_t*>(&timeStamp));
out->setUintField(outValue, i);
break;
}
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
{
string d = DataConvert::dateToString(in.getUintField(i));
out->setStringField(d, i);
break;
}
default:
{
ostringstream os;
os << "TupleUnion::normalize(): tried an illegal conversion: date to "
<< out->getColTypes()[i];
throw logic_error(os.str());
}
}
break;
case CalpontSystemCatalog::DATETIME:
switch (out->getColTypes()[i])
{
case CalpontSystemCatalog::DATETIME:
out->setIntField(in.getIntField(i), i);
break;
case CalpontSystemCatalog::DATE:
{
uint64_t val = in.getUintField(i);
val >>= 32;
out->setUintField(val, i);
break;
}
case CalpontSystemCatalog::TIMESTAMP:
{
uint64_t val = in.getUintField(i);
dataconvert::DateTime dtime(val);
dataconvert::MySQLTime m_time;
dataconvert::TimeStamp timeStamp;
m_time.year = dtime.year;
m_time.month = dtime.month;
m_time.day = dtime.day;
m_time.hour = dtime.hour;
m_time.minute = dtime.minute;
m_time.second = dtime.second;
m_time.second_part = dtime.msecond;
bool isValid = true;
int64_t seconds = mySQLTimeToGmtSec(m_time, fTimeZone, isValid);
if (!isValid)
{
timeStamp.reset();
}
else
{
timeStamp.second = seconds;
timeStamp.msecond = m_time.second_part;
}
uint64_t outValue = (uint64_t) *(reinterpret_cast<uint64_t*>(&timeStamp));
out->setUintField(outValue, i);
break;
}
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
{
string d = DataConvert::datetimeToString(in.getUintField(i));
out->setStringField(d, i);
break;
}
default:
{
ostringstream os;
os << "TupleUnion::normalize(): tried an illegal conversion: datetime to "
<< out->getColTypes()[i];
throw logic_error(os.str());
}
}
break;
case CalpontSystemCatalog::TIMESTAMP:
switch (out->getColTypes()[i])
{
case CalpontSystemCatalog::TIMESTAMP:
out->setIntField(in.getIntField(i), i);
break;
case CalpontSystemCatalog::DATE:
case CalpontSystemCatalog::DATETIME:
{
uint64_t val = in.getUintField(i);
dataconvert::TimeStamp timestamp(val);
int64_t seconds = timestamp.second;
uint64_t outValue;
dataconvert::MySQLTime time;
dataconvert::gmtSecToMySQLTime(seconds, time, fTimeZone);
if (out->getColTypes()[i] == CalpontSystemCatalog::DATE)
{
dataconvert::Date date;
date.year = time.year;
date.month = time.month;
date.day = time.day;
date.spare = 0;
outValue = (uint32_t) *(reinterpret_cast<uint32_t*>(&date));
}
else
{
dataconvert::DateTime datetime;
datetime.year = time.year;
datetime.month = time.month;
datetime.day = time.day;
datetime.hour = time.hour;
datetime.minute = time.minute;
datetime.second = time.second;
datetime.msecond = timestamp.msecond;
outValue = (uint64_t) *(reinterpret_cast<uint64_t*>(&datetime));
}
out->setUintField(outValue, i);
break;
}
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
{
string d = DataConvert::timestampToString(in.getUintField(i), fTimeZone);
out->setStringField(d, i);
break;
}
default:
{
ostringstream os;
os << "TupleUnion::normalize(): tried an illegal conversion: timestamp to "
<< out->getColTypes()[i];
throw logic_error(os.str());
}
}
break;
case CalpontSystemCatalog::TIME:
switch (out->getColTypes()[i])
{
case CalpontSystemCatalog::TIME:
out->setIntField(in.getIntField(i), i);
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
{
string d = DataConvert::timeToString(in.getIntField(i));
out->setStringField(d, i);
break;
}
default:
{
ostringstream os;
os << "TupleUnion::normalize(): tried an illegal conversion: time to "
<< out->getColTypes()[i];
throw logic_error(os.str());
}
}
break;
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
{
double val = (in.getColTypes()[i] == CalpontSystemCatalog::FLOAT ?
in.getFloatField(i) : in.getDoubleField(i));
switch (out->getColTypes()[i])
{
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
case CalpontSystemCatalog::BIGINT:
if (out->getScale(i))
goto dec3;
out->setIntField((int64_t) val, i);
break;
case CalpontSystemCatalog::UTINYINT:
case CalpontSystemCatalog::USMALLINT:
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
case CalpontSystemCatalog::UBIGINT:
out->setUintField((uint64_t) val, i);
break;
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
out->setFloatField(val, i);
break;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
out->setDoubleField(val, i);
break;
case CalpontSystemCatalog::LONGDOUBLE:
out->setLongDoubleField(val, i);
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
{
ostringstream os;
os.precision(15); // to match mysql's output
os << val;
out->setStringField(os.str(), i);
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
dec3: /* have to pick a scale to use for the double. using 5... */
uint32_t scale = 5;
uint64_t ival = (uint64_t) (double) (val * pow((double) 10, (double) scale));
int diff = out->getScale(i) - scale;
if (diff < 0)
ival /= (uint64_t) pow((double) 10, (double) - diff);
else
ival *= (uint64_t) pow((double) 10, (double) diff);
out->setIntField((int64_t) val, i);
break;
}
default:
ostringstream os;
os << "TupleUnion::normalize(): tried an illegal conversion: floating point to "
<< out->getColTypes()[i];
throw logic_error(os.str());
}
break;
}
case CalpontSystemCatalog::LONGDOUBLE:
{
long double val = in.getLongDoubleField(i);
switch (out->getColTypes()[i])
{
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
case CalpontSystemCatalog::BIGINT:
if (out->getScale(i))
goto dec4;
out->setIntField((int64_t) val, i);
break;
case CalpontSystemCatalog::UTINYINT:
case CalpontSystemCatalog::USMALLINT:
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
case CalpontSystemCatalog::UBIGINT:
out->setUintField((uint64_t) val, i);
break;
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
out->setFloatField(val, i);
break;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
out->setDoubleField(val, i);
break;
case CalpontSystemCatalog::LONGDOUBLE:
out->setLongDoubleField(val, i);
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
{
ostringstream os;
os.precision(15); // to match mysql's output
os << val;
out->setStringField(os.str(), i);
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
dec4: /* have to pick a scale to use for the double. using 5... */
uint32_t scale = 5;
uint64_t ival = (uint64_t) (double) (val * pow((double) 10, (double) scale));
int diff = out->getScale(i) - scale;
if (diff < 0)
ival /= (uint64_t) pow((double) 10, (double) - diff);
else
ival *= (uint64_t) pow((double) 10, (double) diff);
out->setIntField((int64_t) val, i);
break;
}
default:
ostringstream os;
os << "TupleUnion::normalize(): tried an illegal conversion: floating point to "
<< out->getColTypes()[i];
throw logic_error(os.str());
}
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
int64_t val = in.getIntField(i);
uint32_t scale = in.getScale(i);
switch (out->getColTypes()[i])
{
case CalpontSystemCatalog::TINYINT:
case CalpontSystemCatalog::SMALLINT:
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
case CalpontSystemCatalog::BIGINT:
case CalpontSystemCatalog::UTINYINT:
case CalpontSystemCatalog::USMALLINT:
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
case CalpontSystemCatalog::UBIGINT:
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
if (out->getScale(i) == scale)
out->setIntField(val, i);
else if (out->getScale(i) > scale)
out->setIntField(IDB_pow[out->getScale(i) - scale]*val, i);
else // should not happen, the output's scale is the largest
throw logic_error("TupleUnion::normalize(): incorrect scale setting");
break;
}
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
{
float fval = ((float) val) / IDB_pow[scale];
out->setFloatField(fval, i);
break;
}
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
{
double dval = ((double) val) / IDB_pow[scale];
out->setDoubleField(dval, i);
break;
}
case CalpontSystemCatalog::LONGDOUBLE:
{
long double dval = ((long double) val) / IDB_pow[scale];
out->setLongDoubleField(dval, i);
break;
}
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
default:
{
char buf[50];
dataconvert::DataConvert::decimalToString(val, scale, buf, 50, out->getColTypes()[i]);
/* ostringstream oss;
if (scale == 0)
oss << val;
else
oss << (val / IDB_pow[scale]) << "."
<< setw(scale) << setfill('0') << (val % IDB_pow[scale]); */
out->setStringField(string(buf), i);
break;
}
}
break;
}
case CalpontSystemCatalog::BLOB:
case CalpontSystemCatalog::VARBINARY:
{
// out->setVarBinaryField(in.getVarBinaryStringField(i), i); // not efficient
out->setVarBinaryField(in.getVarBinaryField(i), in.getVarBinaryLength(i), i);
break;
}
default:
{
ostringstream os;
os << "TupleUnion::normalize(): unknown input type (" << in.getColTypes()[i]
<< ")";
cout << os.str() << endl;
throw logic_error(os.str());
}
}
}
}
void TupleUnion::run()
{
uint32_t i;
mutex::scoped_lock lk(jlLock);
if (runRan)
return;
runRan = true;
lk.unlock();
for (i = 0; i < fInputJobStepAssociation.outSize(); i++)
inputs.push_back(fInputJobStepAssociation.outAt(i)->rowGroupDL());
output = fOutputJobStepAssociation.outAt(0)->rowGroupDL();
if (fDelivery)
{
outputIt = output->getIterator();
}
outputRG.initRow(&row);
outputRG.initRow(&row2);
distinctCount = 0;
normalizedData.reset(new RGData[inputs.size()]);
for (i = 0; i < inputs.size(); i++)
{
if (distinctFlags[i])
{
distinctCount++;
normalizedData[i].reinit(outputRG);
}
}
runners.reserve(inputs.size());
for (i = 0; i < inputs.size(); i++)
{
runners.push_back(jobstepThreadPool.invoke(Runner(this, i)));
}
}
void TupleUnion::join()
{
mutex::scoped_lock lk(jlLock);
if (joinRan)
return;
joinRan = true;
lk.unlock();
jobstepThreadPool.join(runners);
runners.clear();
uniquer->clear();
rowMemory.clear();
rm->returnMemory(memUsage, sessionMemLimit);
memUsage = 0;
}
const string TupleUnion::toString() const
{
ostringstream oss;
oss << "TupleUnion ses:" << fSessionId << " txn:" << fTxnId << " ver:" << fVerId;
oss << " st:" << fStepId;
oss << " in:";
for (unsigned i = 0; i < fInputJobStepAssociation.outSize(); i++)
oss << ((i == 0) ? " " : ", ") << fInputJobStepAssociation.outAt(i);
oss << " out:";
for (unsigned i = 0; i < fOutputJobStepAssociation.outSize(); i++)
oss << ((i == 0) ? " " : ", ") << fOutputJobStepAssociation.outAt(i);
oss << endl;
return oss.str();
}
void TupleUnion::writeNull(Row* out, uint32_t col)
{
switch (out->getColTypes()[col])
{
case CalpontSystemCatalog::TINYINT:
out->setUintField<1>(joblist::TINYINTNULL, col);
break;
case CalpontSystemCatalog::SMALLINT:
out->setUintField<1>(joblist::SMALLINTNULL, col);
break;
case CalpontSystemCatalog::UTINYINT:
out->setUintField<1>(joblist::UTINYINTNULL, col);
break;
case CalpontSystemCatalog::USMALLINT:
out->setUintField<1>(joblist::USMALLINTNULL, col);
break;
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
uint32_t len = out->getColumnWidth(col);
switch (len)
{
case 1:
out->setUintField<1>(joblist::TINYINTNULL, col);
break;
case 2:
out->setUintField<1>(joblist::SMALLINTNULL, col);
break;
case 4:
out->setUintField<4>(joblist::INTNULL, col);
break;
case 8:
out->setUintField<8>(joblist::BIGINTNULL, col);
break;
default:
{}
}
break;
}
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
out->setUintField<4>(joblist::INTNULL, col);
break;
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
out->setUintField<4>(joblist::UINTNULL, col);
break;
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
out->setUintField<4>(joblist::FLOATNULL, col);
break;
case CalpontSystemCatalog::DATE:
out->setUintField<4>(joblist::DATENULL, col);
break;
case CalpontSystemCatalog::BIGINT:
out->setUintField<8>(joblist::BIGINTNULL, col);
break;
case CalpontSystemCatalog::UBIGINT:
out->setUintField<8>(joblist::UBIGINTNULL, col);
break;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
out->setUintField<8>(joblist::DOUBLENULL, col);
break;
case CalpontSystemCatalog::DATETIME:
out->setUintField<8>(joblist::DATETIMENULL, col);
break;
case CalpontSystemCatalog::TIMESTAMP:
out->setUintField<8>(joblist::TIMESTAMPNULL, col);
break;
case CalpontSystemCatalog::TIME:
out->setUintField<8>(joblist::TIMENULL, col);
break;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::TEXT:
case CalpontSystemCatalog::VARCHAR:
{
uint32_t len = out->getColumnWidth(col);
switch (len)
{
case 1:
out->setUintField<1>(joblist::CHAR1NULL, col);
break;
case 2:
out->setUintField<2>(joblist::CHAR2NULL, col);
break;
case 3:
case 4:
out->setUintField<4>(joblist::CHAR4NULL, col);
break;
case 5:
case 6:
case 7:
case 8:
out->setUintField<8>(joblist::CHAR8NULL, col);
break;
default:
out->setStringField(joblist::CPNULLSTRMARK, col);
break;
}
break;
}
case CalpontSystemCatalog::BLOB:
case CalpontSystemCatalog::VARBINARY:
// could use below if zero length and NULL are treated the same
// out->setVarBinaryField("", col); break;
out->setVarBinaryField(joblist::CPNULLSTRMARK, col);
break;
default:
{ }
}
}
void TupleUnion::formatMiniStats()
{
ostringstream oss;
oss << "TUS "
<< "UM "
<< "- "
<< "- "
<< "- "
<< "- "
<< "- "
<< "- "
<< JSTimeStamp::tsdiffstr(dlTimes.EndOfInputTime(), dlTimes.FirstReadTime()) << " "
<< fRowsReturned << " ";
fMiniInfo += oss.str();
}
}
View Git Blame Copy Permalink