database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-09-11 08:50:45 +03:00
Code Activity

Merge pull request #925 from drrtuy/MCOL-894_develop_2

Browse Source 
MCOL-894 parallel sort + MCOL-3536 UTF-8 aware sorting upmerge from 1.2
This commit is contained in:
Andrew Hutchings
2019-11-07 17:26:19 +00:00
committed by GitHub
parent 1254f588a7 0696696cf6
commit bd41ffd6cc
23 changed files with 2077 additions and 361 deletions
Download Patch File Download Diff File Expand all files Collapse all files

222
utils/rowgroup/rowgroup.cpp
View File

@@ -979,117 +979,6 @@ bool Row::isNullValue(uint32_t colIndex) const
uint64_t Row::getNullValue(uint32_t colIndex) const
{
return utils::getNullValue(types[colIndex], getColumnWidth(colIndex));
#if 0
switch (types[colIndex])
{
case CalpontSystemCatalog::TINYINT:
return joblist::TINYINTNULL;
case CalpontSystemCatalog::SMALLINT:
return joblist::SMALLINTNULL;
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
return joblist::INTNULL;
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
return joblist::FLOATNULL;
case CalpontSystemCatalog::DATE:
return joblist::DATENULL;
case CalpontSystemCatalog::BIGINT:
return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME:
return joblist::DATETIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT:
{
uint32_t len = getColumnWidth(colIndex);
switch (len)
{
case 1:
return joblist::CHAR1NULL;
case 2:
return joblist::CHAR2NULL;
case 3:
case 4:
return joblist::CHAR4NULL;
case 5:
case 6:
case 7:
case 8:
return joblist::CHAR8NULL;
default:
throw logic_error("Row::getNullValue() Can't return the NULL string");
}
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
uint32_t len = getColumnWidth(colIndex);
switch (len)
{
case 1 :
return joblist::TINYINTNULL;
case 2 :
return joblist::SMALLINTNULL;
case 4 :
return joblist::INTNULL;
default:
return joblist::BIGINTNULL;
}
break;
}
case CalpontSystemCatalog::UTINYINT:
return joblist::UTINYINTNULL;
case CalpontSystemCatalog::USMALLINT:
return joblist::USMALLINTNULL;
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
return joblist::UINTNULL;
case CalpontSystemCatalog::UBIGINT:
return joblist::UBIGINTNULL;
case CalpontSystemCatalog::LONGDOUBLE:
return -1; // no NULL value for long double yet, this is a nan.
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "Row::getNullValue(): got bad column type (" << types[colIndex] <<
"). Width=" << getColumnWidth(colIndex) << endl;
os << toString() << endl;
throw logic_error(os.str());
}
#endif
}
/* This fcn might produce overflow warnings from the compiler, but that's OK.
@@ -1098,117 +987,6 @@ uint64_t Row::getNullValue(uint32_t colIndex) const
int64_t Row::getSignedNullValue(uint32_t colIndex) const
{
return utils::getSignedNullValue(types[colIndex], getColumnWidth(colIndex));
#if 0
switch (types[colIndex])
{
case CalpontSystemCatalog::TINYINT:
return (int64_t) ((int8_t) joblist::TINYINTNULL);
case CalpontSystemCatalog::SMALLINT:
return (int64_t) ((int16_t) joblist::SMALLINTNULL);
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
return (int64_t) ((int32_t) joblist::INTNULL);
case CalpontSystemCatalog::FLOAT:
case CalpontSystemCatalog::UFLOAT:
return (int64_t) ((int32_t) joblist::FLOATNULL);
case CalpontSystemCatalog::DATE:
return (int64_t) ((int32_t) joblist::DATENULL);
case CalpontSystemCatalog::BIGINT:
return joblist::BIGINTNULL;
case CalpontSystemCatalog::DOUBLE:
case CalpontSystemCatalog::UDOUBLE:
return joblist::DOUBLENULL;
case CalpontSystemCatalog::DATETIME:
return joblist::DATETIMENULL;
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::STRINT:
{
uint32_t len = getColumnWidth(colIndex);
switch (len)
{
case 1:
return (int64_t) ((int8_t) joblist::CHAR1NULL);
case 2:
return (int64_t) ((int16_t) joblist::CHAR2NULL);
case 3:
case 4:
return (int64_t) ((int32_t) joblist::CHAR4NULL);
case 5:
case 6:
case 7:
case 8:
return joblist::CHAR8NULL;
default:
throw logic_error("Row::getSignedNullValue() Can't return the NULL string");
}
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
uint32_t len = getColumnWidth(colIndex);
switch (len)
{
case 1 :
return (int64_t) ((int8_t) joblist::TINYINTNULL);
case 2 :
return (int64_t) ((int16_t) joblist::SMALLINTNULL);
case 4 :
return (int64_t) ((int32_t) joblist::INTNULL);
default:
return joblist::BIGINTNULL;
}
break;
}
case CalpontSystemCatalog::UTINYINT:
return (int64_t) ((int8_t) joblist::UTINYINTNULL);
case CalpontSystemCatalog::USMALLINT:
return (int64_t) ((int16_t) joblist::USMALLINTNULL);
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
return (int64_t) ((int32_t) joblist::UINTNULL);
case CalpontSystemCatalog::UBIGINT:
return (int64_t)joblist::UBIGINTNULL;
case CalpontSystemCatalog::LONGDOUBLE:
return -1; // no NULL value for long double yet, this is a nan.
case CalpontSystemCatalog::VARBINARY:
default:
ostringstream os;
os << "Row::getSignedNullValue(): got bad column type (" << types[colIndex] <<
"). Width=" << getColumnWidth(colIndex) << endl;
os << toString() << endl;
throw logic_error(os.str());
}
#endif
}
RowGroup::RowGroup() : columnCount(0), data(NULL), rgData(NULL), strings(NULL),

2
utils/rowgroup/rowgroup.h
View File

@@ -63,6 +63,8 @@
namespace rowgroup
{
const int16_t rgCommonSize = 8192;
/*
The format of the data RowGroup points to is currently ...

5
utils/windowfunction/CMakeLists.txt
View File

@@ -30,3 +30,8 @@ set_target_properties(windowfunction PROPERTIES VERSION 1.0.0 SOVERSION 1)
install(TARGETS windowfunction DESTINATION ${ENGINE_LIBDIR} COMPONENT libs)
if (WITH_SORTING_COMPARATORS_UT)
add_executable(comparators_tests comparators-tests.cpp)
target_link_libraries(comparators_tests ${ENGINE_LDFLAGS} ${MARIADB_CLIENT_LIBS} ${ENGINE_WRITE_LIBS} ${CPPUNIT_LIBRARIES} cppunit)
install(TARGETS comparators_tests DESTINATION ${ENGINE_BINDIR} COMPONENT platform)
endif()

425
utils/windowfunction/comparators-tests.cpp Normal file
View File

@@ -0,0 +1,425 @@
/* Copyright (C) 2019 MariaDB Corporaton.
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; version 2 of
the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301, USA. */
// $Id: tdriver-filter.cpp 9210 2013-01-21 14:10:42Z rdempsey $
#include <list>
#include <sstream>
#include <pthread.h>
#include <iomanip>
#include <cppunit/extensions/HelperMacros.h>
#include <cppunit/extensions/TestFactoryRegistry.h>
#include <cppunit/ui/text/TestRunner.h>
#include <time.h>
#include <sys/time.h>
#include <limits.h>
#include "jobstep.h"
#include "funcexp.h"
#include "jlf_common.h"
#include "tupleannexstep.h"
#include "calpontsystemcatalog.h"
#include <boost/any.hpp>
#include <boost/function.hpp>
#include "bytestream.h"
#include "idborderby.h"
#define DEBUG
#define MEMORY_LIMIT 14983602176
using namespace std;
using namespace joblist;
using namespace messageqcpp;
// Timer class used by this tdriver to output elapsed times, etc.
class Timer
{
public:
void start(const string& message)
{
if (!fHeaderDisplayed)
{
header();
fHeaderDisplayed = true;
}
gettimeofday(&fTvStart, 0);
cout << timestr() << " Start " << message << endl;
}
void stop(const string& message)
{
time_t now;
time(&now);
string secondsElapsed;
getTimeElapsed(secondsElapsed);
cout << timestr() << " " << secondsElapsed << " Stop " << message << endl;
}
Timer() : fHeaderDisplayed(false) {}
private:
struct timeval fTvStart;
bool fHeaderDisplayed;
double getTimeElapsed(string& seconds)
{
struct timeval tvStop;
gettimeofday(&tvStop, 0);
double secondsElapsed =
(tvStop.tv_sec + (tvStop.tv_usec / 1000000.0)) -
(fTvStart.tv_sec + (fTvStart.tv_usec / 1000000.0));
ostringstream oss;
oss << secondsElapsed;
seconds = oss.str();
seconds.resize(8, '0');
return secondsElapsed;
}
string timestr()
{
struct tm tm;
struct timeval tv;
gettimeofday(&tv, 0);
localtime_r(&tv.tv_sec, &tm);
ostringstream oss;
oss << setfill('0')
<< setw(2) << tm.tm_hour << ':'
<< setw(2) << tm.tm_min << ':'
<< setw(2) << tm.tm_sec << '.'
<< setw(6) << tv.tv_usec
;
return oss.str();
}
void header()
{
cout << endl;
cout << "Time Seconds Activity" << endl;
}
};
class FilterDriver : public CppUnit::TestFixture
{
CPPUNIT_TEST_SUITE(FilterDriver);
CPPUNIT_TEST(INT_TEST);
CPPUNIT_TEST(FLOAT_TEST);
CPPUNIT_TEST_SUITE_END();
private:
// The tests creates an RG with 1 column of the cscDt type
// then initialize RGData. After that it adds two numeric values (v1 < v2)and two NULL.
// Then creates comparator structores and run a number of tests. v1 < v2
void testComparatorWithDT(execplan::CalpontSystemCatalog::ColDataType cscDt,
uint32_t width,
bool generateRandValues)
{
std::cout << std::endl << "------------------------------------------------------------" << std::endl;
uint32_t oid =3001;
std::vector<uint32_t> offsets, roids, tkeys, cscale, cprecision;
std::vector<execplan::CalpontSystemCatalog::ColDataType> types;
offsets.push_back(2); offsets.push_back(2+width);
roids.push_back(oid);
tkeys.push_back(1);
types.push_back(cscDt);
cscale.push_back(0);
cprecision.push_back(20);
rowgroup::RowGroup inRG(1, //column count
offsets, //oldOffset
roids, // column oids
tkeys, //keys
types, // types
cscale, //scale
cprecision, // precision
20, // sTableThreshold
false //useStringTable
);
rowgroup::RGData rgD = rowgroup::RGData(inRG);
inRG.setData(&rgD);
rowgroup::Row r, r1, r2, r3;
inRG.initRow(&r);
uint32_t rowSize = r.getSize();
inRG.getRow(0, &r);
// Sorting spec describes sorting direction and NULL comparision
// preferences
ordering::IdbSortSpec spec = ordering::IdbSortSpec(0, // column index
true, // ascending
true); // NULLs first
std::vector<ordering::IdbSortSpec> specVect;
specVect.push_back(spec);
switch(cscDt)
{
case execplan::CalpontSystemCatalog::UTINYINT:
{
std::cout << "UTINYINT " << std::endl;
r.setUintField<1>(42, 0);
r.nextRow(rowSize);
r.setUintField<1>(43, 0);
r.nextRow(rowSize);
r.setUintField<1>(joblist::UTINYINTNULL, 0);
break;
}
case execplan::CalpontSystemCatalog::USMALLINT:
{
std::cout << "USMALLINT " << std::endl;
r.setUintField<2>(42, 0);
r.nextRow(rowSize);
r.setUintField<2>(43, 0);
r.nextRow(rowSize);
r.setUintField<2>(joblist::USMALLINTNULL, 0);
break;
}
case execplan::CalpontSystemCatalog::UMEDINT:
case execplan::CalpontSystemCatalog::UINT:
{
std::cout << "UINT " << std::endl;
r.setUintField<4>(42, 0);
r.nextRow(rowSize);
r.setUintField<4>(43, 0);
r.nextRow(rowSize);
r.setUintField<4>(joblist::UINTNULL, 0);
break;
}
case execplan::CalpontSystemCatalog::DATETIME:
case execplan::CalpontSystemCatalog::UBIGINT:
{
std::cout << "UBIGINT " << std::endl;
r.setUintField<8>(42, 0);
r.nextRow(rowSize);
r.setUintField<8>(43, 0);
r.nextRow(rowSize);
r.setUintField<8>(joblist::UBIGINTNULL, 0);
break;
}
case execplan::CalpontSystemCatalog::TINYINT:
{
std::cout << "TINYINT " << std::endl;
r.setIntField<1>(42, 0);
r.nextRow(rowSize);
r.setIntField<1>(43, 0);
r.nextRow(rowSize);
r.setIntField<1>(joblist::TINYINTNULL, 0);
break;
}
case execplan::CalpontSystemCatalog::SMALLINT:
{
std::cout << "SMALLINT " << std::endl;
r.setIntField<2>(42, 0);
r.nextRow(rowSize);
r.setIntField<2>(43, 0);
r.nextRow(rowSize);
r.setIntField<2>(joblist::SMALLINTNULL, 0);
break;
}
case execplan::CalpontSystemCatalog::MEDINT:
case execplan::CalpontSystemCatalog::INT:
case execplan::CalpontSystemCatalog::DATE:
{
if (cscDt == execplan::CalpontSystemCatalog::DATE)
std::cout << "DATE" << std::endl;
else
std::cout << "INT " << std::endl;
r.setIntField<4>(42, 0);
r.nextRow(rowSize);
r.setIntField<4>(43, 0);
r.nextRow(rowSize);
if (cscDt == execplan::CalpontSystemCatalog::DATE)
r.setIntField<4>(joblist::DATENULL, 0);
else
r.setIntField<4>(joblist::INTNULL, 0);
break;
}
case execplan::CalpontSystemCatalog::BIGINT:
{
std::cout << "BIGINT " << std::endl;
r.setIntField<8>(42, 0);
r.nextRow(rowSize);
r.setIntField<8>(43, 0);
r.nextRow(rowSize);
r.setIntField<8>(joblist::BIGINTNULL, 0);
break;
}
case execplan::CalpontSystemCatalog::DECIMAL:
case execplan::CalpontSystemCatalog::UDECIMAL:
{
std::cout << "DECIMAL " << std::endl;
switch (width)
{
case 1 :
{
r.setUintField<1>(42, 0);
r.nextRow(rowSize);
r.setUintField<1>(43, 0);
r.nextRow(rowSize);
r.setUintField<1>(joblist::TINYINTNULL, 0);
break;
}
case 2 :
{
r.setUintField<2>(42, 0);
r.nextRow(rowSize);
r.setUintField<2>(43, 0);
r.nextRow(rowSize);
r.setUintField<2>(joblist::SMALLINTNULL, 0);
break;
}
case 4 :
{
r.setUintField<4>(42, 0);
r.nextRow(rowSize);
r.setUintField<4>(43, 0);
r.nextRow(rowSize);
r.setUintField<4>(joblist::INTNULL, 0);
break;
}
default:
{
r.setUintField<8>(42, 0);
r.nextRow(rowSize);
r.setUintField<8>(43, 0);
r.nextRow(rowSize);
r.setUintField<8>(joblist::BIGINTNULL, 0);
break;
}
}
break;
}
case execplan::CalpontSystemCatalog::FLOAT:
case execplan::CalpontSystemCatalog::UFLOAT:
{
std::cout << "FLOAT " << std::endl;
r.setFloatField(42.1, 0);
r.nextRow(rowSize);
r.setFloatField(43.1, 0);
r.nextRow(rowSize);
r.setUintField(joblist::FLOATNULL, 0);
break;
}
case execplan::CalpontSystemCatalog::DOUBLE:
case execplan::CalpontSystemCatalog::UDOUBLE:
{
std::cout << "DOUBLE " << std::endl;
r.setDoubleField(42.1, 0);
r.nextRow(rowSize);
r.setDoubleField(43.1, 0);
r.nextRow(rowSize);
r.setUintField(joblist::DOUBLENULL, 0);
break;
}
case execplan::CalpontSystemCatalog::LONGDOUBLE:
{
r.setLongDoubleField(42.1, 0);
r.nextRow(rowSize);
r.setLongDoubleField(43.1, 0);
r.nextRow(rowSize);
r.setLongDoubleField(joblist::LONGDOUBLENULL, 0);
break;
}
default:
{
break;
}
}
inRG.setRowCount(3);
inRG.initRow(&r1);
inRG.initRow(&r2);
inRG.initRow(&r3);
inRG.getRow(0, &r1);
inRG.getRow(1, &r2);
inRG.getRow(2, &r3);
std::cout<< "r1 " << r1.toString() << " r2 " << r2.toString()
<< " r3 " << r3.toString() << std::endl;
ordering::IdbCompare idbCompare;
idbCompare.initialize(inRG);
ordering::OrderByData odbData = ordering::OrderByData(specVect, inRG);
bool result = odbData(r1.getPointer(), r2.getPointer());
std::cout << r1.toString() << " < " << r2.toString() << " is "
<< ((result) ? "true" : "false") << std::endl;
CPPUNIT_ASSERT(result == true);
result = odbData(r2.getPointer(), r1.getPointer());
std::cout << r2.toString() << " < " << r1.toString() << " is "
<< ((result) ? "true" : "false") << std::endl;
CPPUNIT_ASSERT(result == false);
result = odbData(r2.getPointer(), r2.getPointer());
std::cout << r2.toString() << " < " << r2.toString() << " is "
<< ((result) ? "true" : "false") << std::endl;
CPPUNIT_ASSERT(result == false);
// Compare value with NULL. if spec.fNf then NULLs goes first
result = odbData(r3.getPointer(), r1.getPointer());
std::cout << r3.toString() << " < " << r1.toString() << " is "
<< ((result) ? "true" : "false") << std::endl;
CPPUNIT_ASSERT(result == true);
// Compare NULL with NULL
result = odbData(r3.getPointer(), r1.getPointer());
std::cout << r3.toString() << " < " << r3.toString() << " is "
<< ((result) ? "true" : "false") << std::endl;
CPPUNIT_ASSERT(result == true);
}
void INT_TEST()
{
//bool generateValues = true;
bool fixedValues = false;
testComparatorWithDT(execplan::CalpontSystemCatalog::UTINYINT, 1, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::USMALLINT, 2, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::UMEDINT, 4, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::UBIGINT, 8, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::DATETIME, 8, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::TINYINT, 1, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::SMALLINT, 2, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::MEDINT, 4, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::DATE, 4, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::BIGINT, 8, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::DECIMAL, 8, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::FLOAT, 4, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::DOUBLE, 8, fixedValues);
testComparatorWithDT(execplan::CalpontSystemCatalog::LONGDOUBLE, 8, fixedValues);
}
void FLOAT_TEST()
{
}
};
CPPUNIT_TEST_SUITE_REGISTRATION(FilterDriver);
int main( int argc, char** argv)
{
CppUnit::TextUi::TestRunner runner;
CppUnit::TestFactoryRegistry& registry = CppUnit::TestFactoryRegistry::getRegistry();
runner.addTest( registry.makeTest() );
bool wasSuccessful = runner.run( "", false );
return (wasSuccessful ? 0 : 1);
}

413
utils/windowfunction/idborderby.cpp
View File

@@ -43,32 +43,85 @@ using namespace rowgroup;
#include "idborderby.h"
#include "joblisttypes.h"
namespace ordering
{
int TinyIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
int ret = 0;
int8_t v1 = l->row1().getIntField(fSpec.fIndex);
int8_t v2 = l->row2().getIntField(fSpec.fIndex);
int8_t nullValue = static_cast<int8_t>(joblist::TINYINTNULL);
if (v1 == nullValue || v2 == nullValue)
{
if (v1 != nullValue && v2 == nullValue)
ret = fSpec.fNf;
else if (v1 == nullValue && v2 != nullValue)
ret = -fSpec.fNf;
}
else
{
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
ret = -fSpec.fAsc;
}
return ret;
}
int SmallIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
int ret = 0;
int16_t v1 = l->row1().getIntField(fSpec.fIndex);
int16_t v2 = l->row2().getIntField(fSpec.fIndex);
int16_t nullValue = static_cast<int16_t>(joblist::SMALLINTNULL);
if (v1 == nullValue || v2 == nullValue)
{
if (v1 != nullValue && v2 == nullValue)
ret = fSpec.fNf;
else if (v1 == nullValue && v2 != nullValue)
ret = -fSpec.fNf;
}
else
{
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
ret = -fSpec.fAsc;
}
return ret;
}
int IntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
bool b1 = l->row1().isNullValue(fSpec.fIndex);
bool b2 = l->row2().isNullValue(fSpec.fIndex);
int ret = 0;
int32_t v1 = l->row1().getIntField(fSpec.fIndex);
int32_t v2 = l->row2().getIntField(fSpec.fIndex);
int32_t nullValue = static_cast<int32_t>(joblist::INTNULL);
if (b1 == true || b2 == true)
if (v1 == nullValue || v2 == nullValue)
{
if (b1 == false && b2 == true)
if (v1 != nullValue && v2 == nullValue)
ret = fSpec.fNf;
else if (b1 == true && b2 == false)
else if (v1 == nullValue && v2 != nullValue)
ret = -fSpec.fNf;
}
else
{
int64_t v1 = l->row1().getIntField(fSpec.fIndex);
int64_t v2 = l->row2().getIntField(fSpec.fIndex);
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
@@ -78,29 +131,25 @@ int IntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
return ret;
}
int UintCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
int BigIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
bool b1 = l->row1().isNullValue(fSpec.fIndex);
bool b2 = l->row2().isNullValue(fSpec.fIndex);
int ret = 0;
int64_t v1 = l->row1().getIntField(fSpec.fIndex);
int64_t v2 = l->row2().getIntField(fSpec.fIndex);
int64_t nullValue = static_cast<int64_t>(joblist::BIGINTNULL);
if (b1 == true || b2 == true)
if (v1 == nullValue || v2 == nullValue)
{
if (b1 == false && b2 == true)
if (v1 != nullValue && v2 == nullValue)
ret = fSpec.fNf;
else if (b1 == true && b2 == false)
else if (v1 == nullValue && v2 != nullValue)
ret = -fSpec.fNf;
}
else
{
uint64_t v1 = l->row1().getUintField(fSpec.fIndex);
uint64_t v2 = l->row2().getUintField(fSpec.fIndex);
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
@@ -110,6 +159,116 @@ int UintCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
return ret;
}
int UTinyIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
int ret = 0;
uint8_t v1 = l->row1().getUintField(fSpec.fIndex);
uint8_t v2 = l->row2().getUintField(fSpec.fIndex);
uint8_t nullValue = static_cast<uint8_t>(joblist::UTINYINTNULL);
if (v1 == nullValue || v2 == nullValue)
{
if (v1 != nullValue && v2 == nullValue)
ret = fSpec.fNf;
else if (v1 == nullValue && v2 != nullValue)
ret = -fSpec.fNf;
}
else
{
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
ret = -fSpec.fAsc;
}
return ret;
}
int USmallIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
int ret = 0;
uint16_t v1 = l->row1().getUintField(fSpec.fIndex);
uint16_t v2 = l->row2().getUintField(fSpec.fIndex);
uint16_t nullValue = static_cast<uint16_t>(joblist::USMALLINTNULL);
if (v1 == nullValue || v2 == nullValue)
{
if (v1 != nullValue && v2 == nullValue)
ret = fSpec.fNf;
else if (v1 == nullValue && v2 != nullValue)
ret = -fSpec.fNf;
}
else
{
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
ret = -fSpec.fAsc;
}
return ret;
}
int UIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
int ret = 0;
uint32_t v1 = l->row1().getUintField(fSpec.fIndex);
uint32_t v2 = l->row2().getUintField(fSpec.fIndex);
uint32_t nullValue = static_cast<uint32_t>(joblist::UINTNULL);
if (v1 == nullValue || v2 == nullValue)
{
if (v1 != nullValue && v2 == nullValue)
ret = fSpec.fNf;
else if (v1 == nullValue && v2 != nullValue)
ret = -fSpec.fNf;
}
else
{
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
ret = -fSpec.fAsc;
}
return ret;
}
int UBigIntCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
int ret = 0;
uint64_t v1 = l->row1().getUintField(fSpec.fIndex);
uint64_t v2 = l->row2().getUintField(fSpec.fIndex);
if (v1 == joblist::UBIGINTNULL || v2 == joblist::UBIGINTNULL)
{
if (v1 != joblist::UBIGINTNULL && v2 == joblist::UBIGINTNULL)
ret = fSpec.fNf;
else if (v1 == joblist::UBIGINTNULL && v2 != joblist::UBIGINTNULL)
ret = -fSpec.fNf;
}
else
{
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
ret = -fSpec.fAsc;
}
return ret;
}
int StringCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
@@ -130,41 +289,45 @@ int StringCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
}
else
{
string v1 = l->row1().getStringField(fSpec.fIndex);
string v2 = l->row2().getStringField(fSpec.fIndex);
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
ret = -fSpec.fAsc;
int len1 = l->row1().getStringLength(fSpec.fIndex);
int len2 = l->row2().getStringLength(fSpec.fIndex);
const char* s1 = (const char*)l->row1().getStringPointer(fSpec.fIndex);
const char* s2 = (const char*)l->row2().getStringPointer(fSpec.fIndex);
// For Japanese, coll.compare() may not be as correct as strncmp
if (JPcodePoint)
{
ret = fSpec.fAsc * strncmp(s1, s2, max(len1,len2));
}
else
{
const std::collate<char>& coll = std::use_facet<std::collate<char> >(loc);
ret = fSpec.fAsc * coll.compare(s1, s1+len1, s2, s2+len2);
}
}
return ret;
}
int DoubleCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
bool b1 = l->row1().isNullValue(fSpec.fIndex);
bool b2 = l->row2().isNullValue(fSpec.fIndex);
uint64_t uiv1 = l->row1().getUintField(fSpec.fIndex);
uint64_t uiv2 = l->row2().getUintField(fSpec.fIndex);
int ret = 0;
if (b1 == true || b2 == true)
if (uiv1 == joblist::DOUBLENULL || uiv2 == joblist::DOUBLENULL)
{
if (b1 == false && b2 == true)
if (uiv1 != joblist::DOUBLENULL && uiv2 == joblist::DOUBLENULL)
ret = fSpec.fNf;
else if (b1 == true && b2 == false)
else if (uiv1 == joblist::DOUBLENULL && uiv2 != joblist::DOUBLENULL)
ret = -fSpec.fNf;
}
else
{
double v1 = l->row1().getDoubleField(fSpec.fIndex);
double v2 = l->row2().getDoubleField(fSpec.fIndex);
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
@@ -179,23 +342,22 @@ int FloatCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
l->row1().setData(r1);
l->row2().setData(r2);
bool b1 = l->row1().isNullValue(fSpec.fIndex);
bool b2 = l->row2().isNullValue(fSpec.fIndex);
int32_t iv1 = l->row1().getIntField(fSpec.fIndex);
int32_t iv2 = l->row2().getIntField(fSpec.fIndex);
int32_t nullValue = static_cast<int32_t>(joblist::FLOATNULL);
int ret = 0;
if (b1 == true || b2 == true)
if (iv1 == nullValue || iv2 == nullValue)
{
if (b1 == false && b2 == true)
if (iv1 != nullValue && iv2 == nullValue)
ret = fSpec.fNf;
else if (b1 == true && b2 == false)
else if (iv1 == nullValue && iv2 != nullValue)
ret = -fSpec.fNf;
}
else
{
float v1 = l->row1().getFloatField(fSpec.fIndex);
float v2 = l->row2().getFloatField(fSpec.fIndex);
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
@@ -210,23 +372,20 @@ int LongDoubleCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r
l->row1().setData(r1);
l->row2().setData(r2);
bool b1 = l->row1().isNullValue(fSpec.fIndex);
bool b2 = l->row2().isNullValue(fSpec.fIndex);
long double v1 = l->row1().getLongDoubleField(fSpec.fIndex);
long double v2 = l->row2().getLongDoubleField(fSpec.fIndex);
int ret = 0;
if (b1 == true || b2 == true)
if (v1 == joblist::LONGDOUBLENULL || v2 == joblist::LONGDOUBLENULL)
{
if (b1 == false && b2 == true)
if (v1 != joblist::LONGDOUBLENULL && v2 == joblist::LONGDOUBLENULL)
ret = fSpec.fNf;
else if (b1 == true && b2 == false)
else if (v1 == joblist::LONGDOUBLENULL && v2 != joblist::LONGDOUBLENULL)
ret = -fSpec.fNf;
}
else
{
long double v1 = l->row1().getLongDoubleField(fSpec.fIndex);
long double v2 = l->row2().getLongDoubleField(fSpec.fIndex);
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
@@ -236,29 +395,79 @@ int LongDoubleCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r
return ret;
}
int DateCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
int ret = 0;
uint32_t v1 = l->row1().getUintField(fSpec.fIndex);
uint32_t v2 = l->row2().getUintField(fSpec.fIndex);
uint32_t nullValue = static_cast<uint32_t>(joblist::DATENULL);
if (v1 == nullValue || v2 == nullValue)
{
if (v1 != nullValue && v2 == nullValue)
ret = fSpec.fNf;
else if (v1 == nullValue && v2 != nullValue)
ret = -fSpec.fNf;
}
else
{
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
ret = -fSpec.fAsc;
}
return ret;
}
int DatetimeCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
int ret = 0;
uint64_t v1 = l->row1().getUintField(fSpec.fIndex);
uint64_t v2 = l->row2().getUintField(fSpec.fIndex);
if (v1 == joblist::DATETIMENULL || v2 == joblist::DATETIMENULL)
{
if (v1 != joblist::DATETIMENULL && v2 == joblist::DATETIMENULL)
ret = fSpec.fNf;
else if (v1 == joblist::DATETIMENULL && v2 != joblist::DATETIMENULL)
ret = -fSpec.fNf;
}
else
{
if (v1 > v2)
ret = fSpec.fAsc;
else if (v1 < v2)
ret = -fSpec.fAsc;
}
return ret;
}
int TimeCompare::operator()(IdbCompare* l, Row::Pointer r1, Row::Pointer r2)
{
l->row1().setData(r1);
l->row2().setData(r2);
bool b1 = l->row1().isNullValue(fSpec.fIndex);
bool b2 = l->row2().isNullValue(fSpec.fIndex);
int ret = 0;
uint64_t v1 = l->row1().getUintField(fSpec.fIndex);
uint64_t v2 = l->row2().getUintField(fSpec.fIndex);
if (b1 == true || b2 == true)
if (v1 == joblist::TIMENULL || v2 == joblist::TIMENULL)
{
if (b1 == false && b2 == true)
if (v1 != joblist::TIMENULL && v2 == joblist::TIMENULL)
ret = fSpec.fNf;
else if (b1 == true && b2 == false)
else if (v1 == joblist::TIMENULL && v2 != joblist::TIMENULL)
ret = -fSpec.fNf;
}
else
{
int64_t v1 = l->row1().getIntField(fSpec.fIndex);
int64_t v2 = l->row2().getIntField(fSpec.fIndex);
// ((int64_t) -00:00:26) > ((int64_t) -00:00:25)
// i.e. For 2 negative TIME values, we invert the order of
// comparison operations to force "-00:00:26" to appear before
@@ -301,6 +510,16 @@ bool CompareRule::less(Row::Pointer r1, Row::Pointer r2)
return false;
}
void CompareRule::revertRules()
{
std::vector<Compare*>::iterator fCompareIter = fCompares.begin();
for(; fCompareIter!=fCompares.end(); fCompareIter++)
{
(*fCompareIter)->revertSortSpec();
}
}
void CompareRule::compileRules(const std::vector<IdbSortSpec>& spec, const rowgroup::RowGroup& rg)
{
@@ -311,31 +530,80 @@ void CompareRule::compileRules(const std::vector<IdbSortSpec>& spec, const rowgr
switch (types[i->fIndex])
{
case CalpontSystemCatalog::TINYINT:
{
Compare* c = new TinyIntCompare(*i);
fCompares.push_back(c);
break;
}
case CalpontSystemCatalog::SMALLINT:
{
Compare* c = new SmallIntCompare(*i);
fCompares.push_back(c);
break;
}
case CalpontSystemCatalog::MEDINT:
case CalpontSystemCatalog::INT:
case CalpontSystemCatalog::BIGINT:
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
Compare* c = new IntCompare(*i);
fCompares.push_back(c);
break;
}
case CalpontSystemCatalog::BIGINT:
{
Compare* c = new BigIntCompare(*i);
fCompares.push_back(c);
break;
}
case CalpontSystemCatalog::DECIMAL:
case CalpontSystemCatalog::UDECIMAL:
{
Compare* c;
uint32_t len = rg.getColumnWidth(i->fIndex);
switch (len)
{
case 1 :
c = new TinyIntCompare(*i); break;
case 2 :
c = new SmallIntCompare(*i); break;
case 4 :
c = new IntCompare(*i); break;
default:
c = new BigIntCompare(*i);
}
fCompares.push_back(c);
break;
}
case CalpontSystemCatalog::UTINYINT:
{
Compare* c = new UTinyIntCompare(*i);
fCompares.push_back(c);
break;
}
case CalpontSystemCatalog::USMALLINT:
{
Compare* c = new USmallIntCompare(*i);
fCompares.push_back(c);
break;
}
case CalpontSystemCatalog::UMEDINT:
case CalpontSystemCatalog::UINT:
{
Compare* c = new UIntCompare(*i);
fCompares.push_back(c);
break;
}
case CalpontSystemCatalog::UBIGINT:
{
Compare* c = new UintCompare(*i);
Compare* c = new UBigIntCompare(*i);
fCompares.push_back(c);
break;
}
case CalpontSystemCatalog::CHAR:
case CalpontSystemCatalog::VARCHAR:
case CalpontSystemCatalog::TEXT:
{
Compare* c = new StringCompare(*i);
fCompares.push_back(c);
@@ -366,14 +634,18 @@ void CompareRule::compileRules(const std::vector<IdbSortSpec>& spec, const rowgr
}
case CalpontSystemCatalog::DATE:
case CalpontSystemCatalog::DATETIME:
case CalpontSystemCatalog::TIMESTAMP:
{
Compare* c = new UintCompare(*i);
Compare* c = new DateCompare(*i);
fCompares.push_back(c);
break;
}
case CalpontSystemCatalog::DATETIME:
case CalpontSystemCatalog::TIMESTAMP:
{
Compare* c = new DatetimeCompare(*i);
fCompares.push_back(c);
break;
}
case CalpontSystemCatalog::TIME:
{
Compare* c = new TimeCompare(*i);
@@ -417,9 +689,10 @@ OrderByData::OrderByData(const std::vector<IdbSortSpec>& spec, const rowgroup::R
// IdbOrderBy class implementation
IdbOrderBy::IdbOrderBy() :
fDistinct(false), fMemSize(0), fRowsPerRG(8192), fErrorCode(0), fRm(NULL)
{
}
fDistinct(false), fMemSize(0), fRowsPerRG(rowgroup::rgCommonSize),
fErrorCode(0),
fRm(NULL)
{ }
IdbOrderBy::~IdbOrderBy()

169
utils/windowfunction/idborderby.h
View File

@@ -41,7 +41,6 @@
#include "hasher.h"
#include "stlpoolallocator.h"
// forward reference
namespace joblist
{
@@ -52,9 +51,27 @@ class ResourceManager;
namespace ordering
{
template<typename _Tp, typename _Sequence = std::vector<_Tp>,
typename _Compare = std::less<typename _Sequence::value_type> >
class reservablePQ: private std::priority_queue<_Tp, _Sequence, _Compare>
{
public:
typedef typename std::priority_queue<_Tp, _Sequence, _Compare>::size_type size_type;
reservablePQ(size_type capacity = 0) { reserve(capacity); };
void reserve(size_type capacity) { this->c.reserve(capacity); }
size_type capacity() const { return this->c.capacity(); }
using std::priority_queue<_Tp, _Sequence, _Compare>::size;
using std::priority_queue<_Tp, _Sequence, _Compare>::top;
using std::priority_queue<_Tp, _Sequence, _Compare>::pop;
using std::priority_queue<_Tp, _Sequence, _Compare>::push;
using std::priority_queue<_Tp, _Sequence, _Compare>::empty;
};
// forward reference
class IdbCompare;
class OrderByRow;
typedef reservablePQ<OrderByRow> SortingPQ;
// order by specification
struct IdbSortSpec
@@ -63,6 +80,7 @@ struct IdbSortSpec
// TODO There are three ordering specs since 10.2
int fAsc; // <ordering specification> ::= ASC | DESC
int fNf; // <null ordering> ::= NULLS FIRST | NULLS LAST
std::string fLocale;
IdbSortSpec() : fIndex(-1), fAsc(1), fNf(1) {}
IdbSortSpec(int i, bool b) : fIndex(i), fAsc(b ? 1 : -1), fNf(fAsc) {}
@@ -75,13 +93,72 @@ struct IdbSortSpec
class Compare
{
public:
Compare(const IdbSortSpec& spec) : fSpec(spec) {}
Compare(const IdbSortSpec& spec) : fSpec(spec)
{
// Save off the current Locale in case something goes wrong.
std::string curLocale = setlocale(LC_COLLATE, NULL);
if (spec.fLocale.length() > 0)
{
fLocale = spec.fLocale;
}
else
{
fLocale = curLocale;
}
try
{
std::locale localloc(fLocale.c_str());
loc = localloc;
}
catch(...)
{
fLocale = curLocale;
std::locale localloc(fLocale.c_str());
loc = localloc;
}
if (fLocale.find("ja_JP") != std::string::npos)
{
JPcodePoint = true;
}
else
{
JPcodePoint = false;
}
}
virtual ~Compare() {}
virtual int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer) = 0;
void revertSortSpec()
{
fSpec.fAsc = -fSpec.fAsc;
fSpec.fNf = -fSpec.fNf;
}
protected:
IdbSortSpec fSpec;
std::string fLocale;
std::locale loc;
bool JPcodePoint; // code point ordering (Japanese UTF) flag
};
// Comparators for signed types
class TinyIntCompare : public Compare
{
public:
TinyIntCompare(const IdbSortSpec& spec) : Compare(spec) {}
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
class SmallIntCompare : public Compare
{
public:
SmallIntCompare(const IdbSortSpec& spec) : Compare(spec) {}
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
@@ -94,24 +171,56 @@ public:
};
class UintCompare : public Compare
class BigIntCompare : public Compare
{
public:
UintCompare(const IdbSortSpec& spec) : Compare(spec) {}
BigIntCompare(const IdbSortSpec& spec) : Compare(spec) {}
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
// End of comparators for signed types
// Comparators for unsigned types
class UTinyIntCompare : public Compare
{
public:
UTinyIntCompare(const IdbSortSpec& spec) : Compare(spec) {}
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
class StringCompare : public Compare
class USmallIntCompare : public Compare
{
public:
StringCompare(const IdbSortSpec& spec) : Compare(spec) {}
USmallIntCompare(const IdbSortSpec& spec) : Compare(spec) {}
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
class UIntCompare : public Compare
{
public:
UIntCompare(const IdbSortSpec& spec) : Compare(spec) {}
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
class UBigIntCompare : public Compare
{
public:
UBigIntCompare(const IdbSortSpec& spec) : Compare(spec) {}
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
// end of comparators for unsigned types
// Comparators for float types
class DoubleCompare : public Compare
{
public:
@@ -120,6 +229,7 @@ public:
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
class LongDoubleCompare : public Compare
{
public:
@@ -137,6 +247,26 @@ public:
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
// End of comparators for float types
// Comparators for temporal types
class DateCompare : public Compare
{
public:
DateCompare(const IdbSortSpec& spec) : Compare(spec) {}
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
class DatetimeCompare : public Compare
{
public:
DatetimeCompare(const IdbSortSpec& spec) : Compare(spec) {}
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
class TimeCompare : public Compare
{
@@ -146,6 +276,19 @@ public:
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
// End of comparators for temporal types
// Comparators for non-fixed size types
class StringCompare : public Compare
{
public:
StringCompare(const IdbSortSpec& spec) : Compare(spec) {}
int operator()(IdbCompare*, rowgroup::Row::Pointer, rowgroup::Row::Pointer);
};
// End of comparators for variable sized types
class CompareRule
{
@@ -156,6 +299,7 @@ public:
bool less(rowgroup::Row::Pointer r1, rowgroup::Row::Pointer r2);
void compileRules(const std::vector<IdbSortSpec>&, const rowgroup::RowGroup&);
void revertRules();
std::vector<Compare*> fCompares;
IdbCompare* fIdbCompare;
@@ -197,7 +341,7 @@ public:
return fRule->less(fData, rhs.fData);
}
rowgroup::Row::Pointer fData;
rowgroup::Row::Pointer fData;
CompareRule* fRule;
};
@@ -216,7 +360,6 @@ public:
bool operator()(rowgroup::Row::Pointer, rowgroup::Row::Pointer);
//protected:
std::vector<uint64_t> fIndex;
};
@@ -263,10 +406,18 @@ public:
{
return fDistinct;
}
SortingPQ& getQueue()
{
return fOrderByQueue;
}
CompareRule &getRule()
{
return fRule;
}
SortingPQ fOrderByQueue;
protected:
std::vector<IdbSortSpec> fOrderByCond;
std::priority_queue<OrderByRow> fOrderByQueue;
rowgroup::Row fRow0;
CompareRule fRule;