database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-5021 Code changes based on review feedback.

Browse Source
This commit is contained in:
Gagan Goel
2022-08-01 19:35:55 -04:00
parent 11b7ee2f11
commit cbfdae3481
20 changed files with 407 additions and 435 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
dbcon/ddlpackageproc/createtableprocessor.cpp
View File

@ -274,9 +274,8 @@ keepGoing:
cout << fTxnid.id << " Create table allocOIDs got the starting oid " << fStartingColOID << endl; cout << fTxnid.id << " Create table allocOIDs got the starting oid " << fStartingColOID << endl;
#endif #endif
uint32_t size = numColumns + numDictCols; uint32_t numColumnOids = numColumns + numDictCols;
idbassert(size > 0); numColumnOids += 1; // MCOL-5021
size += 1; // MCOL-5021
if (fStartingColOID < 0) if (fStartingColOID < 0)
{ {
@ -300,7 +299,7 @@ keepGoing:
bytestream << (uint32_t)createTableStmt.fSessionID; bytestream << (uint32_t)createTableStmt.fSessionID;
bytestream << (uint32_t)txnID.id; bytestream << (uint32_t)txnID.id;
bytestream << (uint32_t)fStartingColOID; bytestream << (uint32_t)fStartingColOID;
bytestream << (uint32_t)(fStartingColOID + size); bytestream << (uint32_t)(fStartingColOID + numColumnOids);
bytestream << (uint32_t)createTableStmt.fTableWithAutoi; bytestream << (uint32_t)createTableStmt.fTableWithAutoi;
uint16_t dbRoot; uint16_t dbRoot;
BRM::OID_t sysOid = 1001; BRM::OID_t sysOid = 1001;
@ -543,7 +542,7 @@ keepGoing:
bytestream << (ByteStream::byte)WE_SVR_WRITE_CREATETABLEFILES; bytestream << (ByteStream::byte)WE_SVR_WRITE_CREATETABLEFILES;
bytestream << uniqueId; bytestream << uniqueId;
bytestream << (uint32_t)txnID.id; bytestream << (uint32_t)txnID.id;
bytestream << size; bytestream << numColumnOids;
unsigned colNum = 0; unsigned colNum = 0;
unsigned dictNum = 0; unsigned dictNum = 0;
@ -607,7 +606,7 @@ keepGoing:
++iter; ++iter;
} }
bytestream << (fStartingColOID + size); bytestream << (fStartingColOID + numColumnOids);
bytestream << (uint8_t)execplan::AUX_COL_DATATYPE; bytestream << (uint8_t)execplan::AUX_COL_DATATYPE;
bytestream << (uint8_t) false; bytestream << (uint8_t) false;
bytestream << (uint32_t)execplan::AUX_COL_WIDTH; bytestream << (uint32_t)execplan::AUX_COL_WIDTH;
@ -630,7 +629,7 @@ keepGoing:
} }
// MCOL-5021 // MCOL-5021
oidList.push_back(fStartingColOID + size); oidList.push_back(fStartingColOID + numColumnOids);
try try
{ {
@ -699,9 +698,9 @@ keepGoing:
bytestream.restart(); bytestream.restart();
bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPFILES; bytestream << (ByteStream::byte)WE_SVR_WRITE_DROPFILES;
bytestream << uniqueId; bytestream << uniqueId;
bytestream << (uint32_t)size; bytestream << (uint32_t)numColumnOids;
for (unsigned i = 0; i < size; i++) for (unsigned i = 0; i < numColumnOids; i++)
{ {
bytestream << (uint32_t)(fStartingColOID + i + 1); bytestream << (uint32_t)(fStartingColOID + i + 1);
} }

26
dbcon/execplan/calpontsystemcatalog.cpp
View File

@ -3662,20 +3662,23 @@ CalpontSystemCatalog::OID CalpontSystemCatalog::tableAUXColumnOID(const TableNam
CalpontSelectExecutionPlan::FilterTokenList filterTokenList; CalpontSelectExecutionPlan::FilterTokenList filterTokenList;
CalpontSelectExecutionPlan::ColumnMap colMap; CalpontSelectExecutionPlan::ColumnMap colMap;
static const std::string sysCatSchemaTablePrefix =
CALPONT_SCHEMA + "." + SYSTABLE_TABLE + ".";
SimpleColumn* c1 = SimpleColumn* c1 =
new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + AUXCOLUMNOID_COL, fSessionID); new SimpleColumn(sysCatSchemaTablePrefix + AUXCOLUMNOID_COL, fSessionID);
SimpleColumn* c2 = SimpleColumn* c2 =
new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, fSessionID); new SimpleColumn(sysCatSchemaTablePrefix + SCHEMA_COL, fSessionID);
SimpleColumn* c3 = SimpleColumn* c3 =
new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, fSessionID); new SimpleColumn(sysCatSchemaTablePrefix + TABLENAME_COL, fSessionID);
SRCP srcp; SRCP srcp;
srcp.reset(c1); srcp.reset(c1);
colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + AUXCOLUMNOID_COL, srcp)); colMap.insert(CMVT_(sysCatSchemaTablePrefix + AUXCOLUMNOID_COL, srcp));
srcp.reset(c2); srcp.reset(c2);
colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + SCHEMA_COL, srcp)); colMap.insert(CMVT_(sysCatSchemaTablePrefix + SCHEMA_COL, srcp));
srcp.reset(c3); srcp.reset(c3);
colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + TABLENAME_COL, srcp)); colMap.insert(CMVT_(sysCatSchemaTablePrefix + TABLENAME_COL, srcp));
csep.columnMapNonStatic(colMap); csep.columnMapNonStatic(colMap);
srcp.reset(c1->clone()); srcp.reset(c1->clone());
@ -3771,15 +3774,18 @@ CalpontSystemCatalog::OID CalpontSystemCatalog::isAUXColumnOID(const OID& oid)
CalpontSelectExecutionPlan::FilterTokenList filterTokenList; CalpontSelectExecutionPlan::FilterTokenList filterTokenList;
CalpontSelectExecutionPlan::ColumnMap colMap; CalpontSelectExecutionPlan::ColumnMap colMap;
static const std::string sysCatSchemaTablePrefix =
CALPONT_SCHEMA + "." + SYSTABLE_TABLE + ".";
SimpleColumn* c1 = SimpleColumn* c1 =
new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, fSessionID); new SimpleColumn(sysCatSchemaTablePrefix + OBJECTID_COL, fSessionID);
SimpleColumn* c2 = SimpleColumn* c2 =
new SimpleColumn(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + AUXCOLUMNOID_COL, fSessionID); new SimpleColumn(sysCatSchemaTablePrefix + AUXCOLUMNOID_COL, fSessionID);
SRCP srcp; SRCP srcp;
srcp.reset(c1); srcp.reset(c1);
colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + OBJECTID_COL, srcp)); colMap.insert(CMVT_(sysCatSchemaTablePrefix + OBJECTID_COL, srcp));
srcp.reset(c2); srcp.reset(c2);
colMap.insert(CMVT_(CALPONT_SCHEMA + "." + SYSTABLE_TABLE + "." + AUXCOLUMNOID_COL, srcp)); colMap.insert(CMVT_(sysCatSchemaTablePrefix + AUXCOLUMNOID_COL, srcp));
csep.columnMapNonStatic(colMap); csep.columnMapNonStatic(colMap);
srcp.reset(c1->clone()); srcp.reset(c1->clone());

7
dbcon/joblist/columncommand-jl.cpp
View File

@ -265,6 +265,11 @@ void ColumnCommandJL::setLBID(uint64_t rid, uint32_t dbRoot)
} }
} }
if (i == extents.size())
{
throw logic_error("ColumnCommandJL: setLBID didn't find the extent for the rid.");
}
uint32_t j; uint32_t j;
for (j = 0; j < extentsAux.size(); j++) for (j = 0; j < extentsAux.size(); j++)
@ -277,7 +282,7 @@ void ColumnCommandJL::setLBID(uint64_t rid, uint32_t dbRoot)
} }
} }
if (i == extents.size() || (hasAuxCol && j == extentsAux.size())) if (hasAuxCol && j == extentsAux.size())
{ {
throw logic_error("ColumnCommandJL: setLBID didn't find the extent for the rid."); throw logic_error("ColumnCommandJL: setLBID didn't find the extent for the rid.");
} }

1
dbcon/joblist/primitivemsg.h
View File

@ -717,6 +717,7 @@ struct NewColRequestHeader
uint16_t NOPS; uint16_t NOPS;
uint16_t NVALS; uint16_t NVALS;
uint8_t sort; // 1 to sort uint8_t sort; // 1 to sort
bool hasAuxCol;
// this follows the header // this follows the header
// ColArgs ArgList[NOPS] (where the val field is DataSize bytes long) // ColArgs ArgList[NOPS] (where the val field is DataSize bytes long)
// uint16_t Rids[NVALS] (each rid is relative to the given block) // uint16_t Rids[NVALS] (each rid is relative to the given block)

2
dbcon/joblist/tuple-bps.cpp
View File

@ -579,7 +579,6 @@ TupleBPS::TupleBPS(const pColScanStep& rhs, const JobInfo& jobInfo) : BatchPrimi
sort(extentsAux.begin(), extentsAux.end(), BRM::ExtentSorter()); sort(extentsAux.begin(), extentsAux.end(), BRM::ExtentSorter());
extentsMap[fOidAux] = tr1::unordered_map<int64_t, EMEntry>();
tr1::unordered_map<int64_t, EMEntry>& refAux = extentsMap[fOidAux]; tr1::unordered_map<int64_t, EMEntry>& refAux = extentsMap[fOidAux];
for (uint32_t z = 0; z < extentsAux.size(); z++) for (uint32_t z = 0; z < extentsAux.size(); z++)
@ -589,7 +588,6 @@ TupleBPS::TupleBPS(const pColScanStep& rhs, const JobInfo& jobInfo) : BatchPrimi
/* These lines are obsoleted by initExtentMarkers. Need to remove & retest. */ /* These lines are obsoleted by initExtentMarkers. Need to remove & retest. */
scannedExtents = rhs.extents; scannedExtents = rhs.extents;
extentsMap[fOid] = tr1::unordered_map<int64_t, EMEntry>();
tr1::unordered_map<int64_t, EMEntry>& ref = extentsMap[fOid]; tr1::unordered_map<int64_t, EMEntry>& ref = extentsMap[fOid];
for (uint32_t z = 0; z < rhs.extents.size(); z++) for (uint32_t z = 0; z < rhs.extents.size(); z++)

2
oam/etc/Columnstore.xml
View File

@ -137,7 +137,7 @@
<BulkRollbackDir>/var/lib/columnstore/data1/systemFiles/bulkRollback</BulkRollbackDir> <BulkRollbackDir>/var/lib/columnstore/data1/systemFiles/bulkRollback</BulkRollbackDir>
<MaxFileSystemDiskUsagePct>98</MaxFileSystemDiskUsagePct> <MaxFileSystemDiskUsagePct>98</MaxFileSystemDiskUsagePct>
<CompressedPaddingBlocks>1</CompressedPaddingBlocks> <!-- Number of blocks used to pad compressed chunks --> <CompressedPaddingBlocks>1</CompressedPaddingBlocks> <!-- Number of blocks used to pad compressed chunks -->
<FastDelete>false</FastDelete> <FastDelete>n</FastDelete>
</WriteEngine> </WriteEngine>
<DBRM_Controller> <DBRM_Controller>
<NumWorkers>1</NumWorkers> <NumWorkers>1</NumWorkers>

546
primitives/linux-port/column.cpp
View File

@ -136,6 +136,44 @@ inline MT getNonEmptyMask16Byte(MT* nonEmptyMaskAux, uint16_t iter)
return nonEmptyMask16Byte[(nonEmptyMaskAux[iter >> 4] >> (iter & 0x000F)) & 0x0001]; return nonEmptyMask16Byte[(nonEmptyMaskAux[iter >> 4] >> (iter & 0x000F)) & 0x0001];
} }
typedef MT (*getNonEmptyMaskPtrT)(MT*, uint16_t);
template <uint16_t WIDTH>
constexpr getNonEmptyMaskPtrT getNonEmptyMaskPtrTemplate()
{
return nullptr;
}
template<>
constexpr getNonEmptyMaskPtrT getNonEmptyMaskPtrTemplate<1>()
{
return getNonEmptyMask1Byte;
}
template<>
constexpr getNonEmptyMaskPtrT getNonEmptyMaskPtrTemplate<2>()
{
return getNonEmptyMask2Byte;
}
template<>
constexpr getNonEmptyMaskPtrT getNonEmptyMaskPtrTemplate<4>()
{
return getNonEmptyMask4Byte;
}
template<>
constexpr getNonEmptyMaskPtrT getNonEmptyMaskPtrTemplate<8>()
{
return getNonEmptyMask8Byte;
}
template<>
constexpr getNonEmptyMaskPtrT getNonEmptyMaskPtrTemplate<16>()
{
return getNonEmptyMask16Byte;
}
inline uint64_t order_swap(uint64_t x) inline uint64_t order_swap(uint64_t x)
{ {
uint64_t ret = (x >> 56) | ((x << 40) & 0x00FF000000000000ULL) | ((x << 24) & 0x0000FF0000000000ULL) | uint64_t ret = (x >> 56) | ((x << 40) & 0x00FF000000000000ULL) | ((x << 24) & 0x0000FF0000000000ULL) |
@ -909,24 +947,22 @@ T getInitialMax(NewColRequestHeader* in)
// Return true on success, false on End of Block. // Return true on success, false on End of Block.
// Values are read from srcArray either in natural order or in the order defined by ridArray. // Values are read from srcArray either in natural order or in the order defined by ridArray.
// Empty values are skipped, unless ridArray==0 && !(OutputType & OT_RID). // Empty values are skipped, unless ridArray==0 && !(OutputType & OT_RID).
template <typename T, int COL_WIDTH> template <typename T, int COL_WIDTH, bool IS_AUX_COLUMN>
inline bool nextColValue( inline bool nextColValue(
T& result, // Place for the value returned T& result, // Place for the value returned
bool* isEmpty, // ... and flag whether it's EMPTY bool* isEmpty, // ... and flag whether it's EMPTY
uint32_t* uint32_t* index, // Successive index either in srcArray (going from 0 to srcSize-1) or ridArray (0..ridSize-1)
index, // Successive index either in srcArray (going from 0 to srcSize-1) or ridArray (0..ridSize-1)
uint16_t* rid, // Index in srcArray of the value returned uint16_t* rid, // Index in srcArray of the value returned
const T* srcArray, // Input array const T* srcArray, // Input array
const uint32_t srcSize, // ... and its size const uint32_t srcSize, // ... and its size
const uint16_t* ridArray, // Optional array of indexes into srcArray, that defines the read order const uint16_t* ridArray, // Optional array of indexes into srcArray, that defines the read order
const uint16_t ridSize, // ... and its size const uint16_t ridSize, // ... and its size
const uint8_t OutputType, // Used to decide whether to skip EMPTY values const uint8_t OutputType, // Used to decide whether to skip EMPTY values
T EMPTY_VALUE, const T& EMPTY_VALUE,
const uint8_t* blockAux, const uint8_t* blockAux,
uint8_t EMPTY_VALUE_AUX) uint8_t EMPTY_VALUE_AUX)
{ {
auto i = *index; // local copy of *index to speed up loops auto i = *index; // local copy of *index to speed up loops
T value; // value to be written into *result, local for the same reason
if (ridArray) if (ridArray)
{ {
@ -936,10 +972,20 @@ inline bool nextColValue(
if (UNLIKELY(i >= ridSize)) if (UNLIKELY(i >= ridSize))
return false; return false;
value = srcArray[ridArray[i]]; if constexpr (IS_AUX_COLUMN)
{
uint8_t valueAux = blockAux[ridArray[i]];
if (value != EMPTY_VALUE) if (valueAux != EMPTY_VALUE_AUX)
break; break;
}
else
{
T value = srcArray[ridArray[i]];
if (value != EMPTY_VALUE)
break;
}
} }
*rid = ridArray[i]; *rid = ridArray[i];
@ -953,10 +999,20 @@ inline bool nextColValue(
if (UNLIKELY(i >= srcSize)) if (UNLIKELY(i >= srcSize))
return false; return false;
value = srcArray[i]; if constexpr (IS_AUX_COLUMN)
{
uint8_t valueAux = blockAux[i];
if (value != EMPTY_VALUE) if (valueAux != EMPTY_VALUE_AUX)
break; break;
}
else
{
T value = srcArray[i];
if (value != EMPTY_VALUE)
break;
}
} }
*rid = i; *rid = i;
@ -969,77 +1025,17 @@ inline bool nextColValue(
return false; return false;
*rid = i; *rid = i;
value = srcArray[i];
*isEmpty = (value == EMPTY_VALUE);
}
*index = i + 1; if constexpr (IS_AUX_COLUMN)
result = value;
return true;
}
template <typename T, int COL_WIDTH>
inline bool nextColValueAux(
T& result, // Place for the value returned
bool* isEmpty, // ... and flag whether it's EMPTY
uint32_t*
index, // Successive index either in srcArray (going from 0 to srcSize-1) or ridArray (0..ridSize-1)
uint16_t* rid, // Index in srcArray of the value returned
const T* srcArray, // Input array
const uint32_t srcSize, // ... and its size
const uint16_t* ridArray, // Optional array of indexes into srcArray, that defines the read order
const uint16_t ridSize, // ... and its size
const uint8_t OutputType, // Used to decide whether to skip EMPTY values
T EMPTY_VALUE,
const uint8_t* blockAux,
uint8_t EMPTY_VALUE_AUX)
{
auto i = *index; // local copy of *index to speed up loops
uint8_t valueAux;
if (ridArray)
{
// Read next non-empty value in the order defined by ridArray
for (;; i++)
{ {
if (UNLIKELY(i >= ridSize)) uint8_t valueAux = blockAux[i];
return false; *isEmpty = (valueAux == EMPTY_VALUE_AUX);
valueAux = blockAux[ridArray[i]];
if (valueAux != EMPTY_VALUE_AUX)
break;
} }
else
*rid = ridArray[i];
*isEmpty = false;
}
else if (OutputType & OT_RID) // TODO: check correctness of this condition for SKIP_EMPTY_VALUES
{
// Read next non-empty value in the natural order
for (;; i++)
{ {
if (UNLIKELY(i >= srcSize)) T value = srcArray[i];
return false; *isEmpty = (value == EMPTY_VALUE);
valueAux = blockAux[i];
if (valueAux != EMPTY_VALUE_AUX)
break;
} }
*rid = i;
*isEmpty = false;
}
else
{
// Read next value in the natural order
if (UNLIKELY(i >= srcSize))
return false;
*rid = i;
valueAux = blockAux[i];
*isEmpty = (valueAux == EMPTY_VALUE_AUX);
} }
*index = i + 1; *index = i + 1;
@ -1330,7 +1326,7 @@ void scalarFiltering(
T emptyValue, // Deduced empty value magic T emptyValue, // Deduced empty value magic
T nullValue, // Deduced null value magic T nullValue, // Deduced null value magic
T Min, T Max, const bool isNullValueMatches, T Min, T Max, const bool isNullValueMatches,
const bool hasAuxCol, const uint8_t* blockAux, const uint8_t* blockAux,
uint8_t emptyValueAux) uint8_t emptyValueAux)
{ {
constexpr int WIDTH = sizeof(T); constexpr int WIDTH = sizeof(T);
@ -1339,7 +1335,7 @@ void scalarFiltering(
primitives::RIDType rid = 0; primitives::RIDType rid = 0;
bool isEmpty = false; bool isEmpty = false;
auto nextColValuePtr = hasAuxCol ? nextColValueAux<T, WIDTH> : nextColValue<T, WIDTH>; auto nextColValuePtr = in->hasAuxCol ? nextColValue<T, WIDTH, true> : nextColValue<T, WIDTH, false>;
// Loop over the column values, storing those matching the filter, and updating the min..max range // Loop over the column values, storing those matching the filter, and updating the min..max range
for (uint32_t i = initialRID; (*nextColValuePtr)(curValue, &isEmpty, &i, &rid, srcArray, srcSize, for (uint32_t i = initialRID; (*nextColValuePtr)(curValue, &isEmpty, &i, &rid, srcArray, srcSize,
@ -1489,6 +1485,34 @@ void extractTextMinMax(VT& simdProcessor, SimdType simdMin, SimdType simdMax, Si
max = simdMaxVec[indMax - weightsMaxVec]; max = simdMaxVec[indMax - weightsMaxVec];
} }
template<bool HAS_INPUT_RIDS>
void buildAuxColEmptyVal(const uint16_t iterNumberAux, const uint16_t vectorSizeAux,
const uint8_t** blockAux, uint8_t emptyValueAux,
MT** nonEmptyMaskAux, primitives::RIDType** ridArray)
{
using SimdTypeTemp = typename simd::IntegralToSIMD<uint8_t, KIND_UNSIGNED>::type;
using FilterTypeTemp = typename simd::StorageToFiltering<uint8_t, KIND_UNSIGNED>::type;
using VTAux = typename simd::SimdFilterProcessor<SimdTypeTemp, FilterTypeTemp>;
using SimdTypeAux = typename VTAux::SimdType;
using SimdWrapperTypeAux = typename VTAux::SimdWrapperType;
VTAux simdProcessorAux;
SimdTypeAux dataVecAux;
SimdTypeAux emptyFilterArgVecAux = simdProcessorAux.emptyNullLoadValue(emptyValueAux);
const uint8_t* origBlockAux = *blockAux;
primitives::RIDType* origRidArray = *ridArray;
for (uint16_t i = 0; i < iterNumberAux; ++i)
{
dataVecAux = simdDataLoad<VTAux, SimdWrapperTypeAux, HAS_INPUT_RIDS, uint8_t>(simdProcessorAux, *blockAux,
origBlockAux, *ridArray, i).v;
(*nonEmptyMaskAux)[i] = simdProcessorAux.nullEmptyCmpNe(dataVecAux, emptyFilterArgVecAux);
*blockAux += vectorSizeAux;
*ridArray += vectorSizeAux;
}
*ridArray = origRidArray;
}
// This routine filters input block in a vectorized manner. // This routine filters input block in a vectorized manner.
// It supports all output types, all input types. // It supports all output types, all input types.
// It doesn't support KIND==TEXT so upper layers filters this KIND out beforehand. // It doesn't support KIND==TEXT so upper layers filters this KIND out beforehand.
@ -1499,12 +1523,12 @@ void extractTextMinMax(VT& simdProcessor, SimdType simdMin, SimdType simdMax, Si
// Then it takes a vector of data, run filters and logical function using pointers. // Then it takes a vector of data, run filters and logical function using pointers.
// See the corresponding dispatcher to get more details on vector processing class. // See the corresponding dispatcher to get more details on vector processing class.
template<typename T, typename VT, bool HAS_INPUT_RIDS, int OUTPUT_TYPE, template<typename T, typename VT, bool HAS_INPUT_RIDS, int OUTPUT_TYPE,
ENUM_KIND KIND, typename FT, typename ST> ENUM_KIND KIND, typename FT, typename ST, bool IS_AUX_COLUMN>
void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* srcArray, void vectorizedFiltering_(NewColRequestHeader* in, ColResultHeader* out, const T* srcArray,
const uint32_t srcSize, primitives::RIDType* ridArray, const uint16_t ridSize, const uint32_t srcSize, primitives::RIDType* ridArray, const uint16_t ridSize,
ParsedColumnFilter* parsedColumnFilter, const bool validMinMax, const T emptyValue, ParsedColumnFilter* parsedColumnFilter, const bool validMinMax, const T emptyValue,
const T nullValue, T min, T max, const bool isNullValueMatches, const T nullValue, T min, T max, const bool isNullValueMatches,
const bool hasAuxCol, const uint8_t* blockAux, uint8_t emptyValueAux) const uint8_t* blockAux, uint8_t emptyValueAux)
{ {
constexpr const uint16_t WIDTH = sizeof(T); constexpr const uint16_t WIDTH = sizeof(T);
using SimdType = typename VT::SimdType; using SimdType = typename VT::SimdType;
@ -1516,7 +1540,7 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T*
SimdType dataVec; SimdType dataVec;
[[maybe_unused]] SimdType swapedOrderDataVec; [[maybe_unused]] SimdType swapedOrderDataVec;
[[maybe_unused]] auto typeHolder = in->colType; [[maybe_unused]] auto typeHolder = in->colType;
SimdType emptyFilterArgVec = simdProcessor.emptyNullLoadValue(emptyValue); [[maybe_unused]] SimdType emptyFilterArgVec = simdProcessor.emptyNullLoadValue(emptyValue);
SimdType nullFilterArgVec = simdProcessor.emptyNullLoadValue(nullValue); SimdType nullFilterArgVec = simdProcessor.emptyNullLoadValue(nullValue);
MT writeMask, nonEmptyMask, nonNullMask, nonNullOrEmptyMask; MT writeMask, nonEmptyMask, nonNullMask, nonNullOrEmptyMask;
MT initFilterMask = 0xFFFF; MT initFilterMask = 0xFFFF;
@ -1637,177 +1661,77 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T*
weightsMax = simdSwapedOrderDataLoad<KIND, VT, SimdWrapperType, T>(typeHolder, simdProcessor, simdMax).v; weightsMax = simdSwapedOrderDataLoad<KIND, VT, SimdWrapperType, T>(typeHolder, simdProcessor, simdMax).v;
} }
if (hasAuxCol) MT* nonEmptyMaskAux;
if constexpr (IS_AUX_COLUMN)
{ {
using SimdTypeTemp = typename simd::IntegralToSIMD<uint8_t, KIND_UNSIGNED>::type; constexpr uint16_t vectorSizeAux = VT::vecByteSize;
using FilterTypeTemp = typename simd::StorageToFiltering<uint8_t, KIND_UNSIGNED>::type; uint16_t iterNumberAux = HAS_INPUT_RIDS ? ridSize / vectorSizeAux : srcSize / vectorSizeAux;
using VTAux = typename simd::SimdFilterProcessor<SimdTypeTemp, FilterTypeTemp>; nonEmptyMaskAux = (MT*) alloca(sizeof(MT) * iterNumberAux);
using SimdTypeAux = typename VTAux::SimdType; buildAuxColEmptyVal<HAS_INPUT_RIDS>(iterNumberAux, vectorSizeAux, &blockAux, emptyValueAux,
using SimdWrapperTypeAux = typename VTAux::SimdWrapperType; &nonEmptyMaskAux, &ridArray);
VTAux simdProcessorAux;
SimdTypeAux dataVecAux;
SimdTypeAux emptyFilterArgVecAux = simdProcessorAux.emptyNullLoadValue(emptyValueAux);
const uint8_t* origBlockAux = blockAux;
constexpr uint16_t VECTOR_SIZE_AUX = VT::vecByteSize;
uint16_t iterNumberAux = HAS_INPUT_RIDS ? ridSize / VECTOR_SIZE_AUX : srcSize / VECTOR_SIZE_AUX;
MT* nonEmptyMaskAux = (MT*) alloca(sizeof(MT) * iterNumberAux);
primitives::RIDType* origRidArray = ridArray;
for (uint16_t i = 0; i < iterNumberAux; ++i)
{
dataVecAux = simdDataLoad<VTAux, SimdWrapperTypeAux, HAS_INPUT_RIDS, uint8_t>(simdProcessorAux, blockAux,
origBlockAux, ridArray, i).v;
nonEmptyMaskAux[i] = simdProcessorAux.nullEmptyCmpNe(dataVecAux, emptyFilterArgVecAux);
blockAux += VECTOR_SIZE_AUX;
ridArray += VECTOR_SIZE_AUX;
}
ridArray = origRidArray;
MT (*getNonEmptyMaskPtr)(MT*, uint16_t);
switch(WIDTH)
{
case 1:
getNonEmptyMaskPtr = getNonEmptyMask1Byte;
break;
case 2:
getNonEmptyMaskPtr = getNonEmptyMask2Byte;
break;
case 4:
getNonEmptyMaskPtr = getNonEmptyMask4Byte;
break;
case 8:
getNonEmptyMaskPtr = getNonEmptyMask8Byte;
break;
case 16:
getNonEmptyMaskPtr = getNonEmptyMask16Byte;
break;
}
// main loop
// writeMask tells which values must get into the result. Includes values that matches filters. Can have
// NULLs. nonEmptyMask tells which vector coords are not EMPTY magics. nonNullMask tells which vector coords
// are not NULL magics.
for (uint16_t i = 0; i < iterNumber; ++i)
{
primitives::RIDType ridOffset = i * VECTOR_SIZE;
assert(!HAS_INPUT_RIDS || (HAS_INPUT_RIDS && ridSize >= ridOffset));
dataVec = simdDataLoad<VT, SimdWrapperType, HAS_INPUT_RIDS, T>(simdProcessor, srcArray,
origSrcArray, ridArray, i).v;
if constexpr(KIND==KIND_TEXT)
swapedOrderDataVec = simdSwapedOrderDataLoad<KIND, VT, SimdWrapperType, T>(typeHolder, simdProcessor, dataVec).v;
nonEmptyMask = (*getNonEmptyMaskPtr)(nonEmptyMaskAux, i);
writeMask = nonEmptyMask;
// NULL check
nonNullMask = simdProcessor.nullEmptyCmpNe(dataVec, nullFilterArgVec);
// Exclude NULLs from the resulting set if NULL doesn't match the filters.
writeMask = isNullValueMatches ? writeMask : writeMask & nonNullMask;
nonNullOrEmptyMask = nonNullMask & nonEmptyMask;
// filters
MT prevFilterMask = initFilterMask;
// TODO name this mask literal
MT filterMask = 0xFFFF;
for (uint32_t j = 0; j < filterCount; ++j)
{
// filter using compiled filter and preloaded filter argument
if constexpr(KIND==KIND_TEXT)
filterMask = copFunctorVec[j](simdProcessor, swapedOrderDataVec, filterArgsVectors[j]);
else
filterMask = copFunctorVec[j](simdProcessor, dataVec, filterArgsVectors[j]);
filterMask = bopFunctor(prevFilterMask, filterMask);
prevFilterMask = filterMask;
}
writeMask = writeMask & filterMask;
T* dataVecTPtr = reinterpret_cast<T*>(&dataVec);
// vectWriteColValues iterates over the values in the source vec
// to store values/RIDs into dstArray/ridDstArray.
// It also sets min/max values for the block if eligible.
// !!! vectWriteColValues increases ridDstArray internally but it doesn't go
// outside the scope of the memory allocated to out msg.
// vectWriteColValues is empty if outputMode == OT_RID.
uint16_t valuesWritten = vectWriteColValues<T, VT, OUTPUT_TYPE, KIND, HAS_INPUT_RIDS>(
simdProcessor, writeMask, nonNullOrEmptyMask, validMinMax, ridOffset, dataVecTPtr, dstArray, min, max,
in, out, ridDstArray, ridArray);
// Some outputType modes saves RIDs also. vectWriteRIDValues is empty for
// OT_DATAVALUE, OT_BOTH(vectWriteColValues takes care about RIDs).
valuesWritten = vectWriteRIDValues<T, VT, OUTPUT_TYPE, KIND, HAS_INPUT_RIDS>(
simdProcessor, valuesWritten, validMinMax, ridOffset, dataVecTPtr, ridDstArray, writeMask, min, max,
in, out, nonNullOrEmptyMask, ridArray);
if constexpr (KIND != KIND_TEXT)
vectorizedUpdateMinMax(validMinMax, nonNullOrEmptyMask, simdProcessor, dataVec, simdMin, simdMax);
else
vectorizedTextUpdateMinMax(validMinMax, nonNullOrEmptyMask, simdProcessor, dataVec, simdMin, simdMax,
swapedOrderDataVec, weightsMin, weightsMax);
// Calculate bytes written
uint16_t bytesWritten = valuesWritten * WIDTH;
totalValuesWritten += valuesWritten;
ridDstArray += valuesWritten;
dstArray += bytesWritten;
rid += VECTOR_SIZE;
srcArray += VECTOR_SIZE;
ridArray += VECTOR_SIZE;
}
} }
else
constexpr getNonEmptyMaskPtrT getNonEmptyMaskPtr = getNonEmptyMaskPtrTemplate<WIDTH>();
// main loop
// writeMask tells which values must get into the result. Includes values that matches filters. Can have
// NULLs. nonEmptyMask tells which vector coords are not EMPTY magics. nonNullMask tells which vector coords
// are not NULL magics.
for (uint16_t i = 0; i < iterNumber; ++i)
{ {
// main loop primitives::RIDType ridOffset = i * VECTOR_SIZE;
// writeMask tells which values must get into the result. Includes values that matches filters. Can have assert(!HAS_INPUT_RIDS || (HAS_INPUT_RIDS && ridSize >= ridOffset));
// NULLs. nonEmptyMask tells which vector coords are not EMPTY magics. nonNullMask tells which vector coords dataVec = simdDataLoad<VT, SimdWrapperType, HAS_INPUT_RIDS, T>(simdProcessor, srcArray,
// are not NULL magics. origSrcArray, ridArray, i).v;
for (uint16_t i = 0; i < iterNumber; ++i)
{ if constexpr(KIND==KIND_TEXT)
primitives::RIDType ridOffset = i * VECTOR_SIZE; swapedOrderDataVec = simdSwapedOrderDataLoad<KIND, VT, SimdWrapperType, T>(typeHolder, simdProcessor, dataVec).v;
assert(!HAS_INPUT_RIDS || (HAS_INPUT_RIDS && ridSize >= ridOffset));
dataVec = simdDataLoad<VT, SimdWrapperType, HAS_INPUT_RIDS, T>(simdProcessor, srcArray, if constexpr (IS_AUX_COLUMN)
origSrcArray, ridArray, i).v; nonEmptyMask = (*getNonEmptyMaskPtr)(nonEmptyMaskAux, i);
if constexpr(KIND==KIND_TEXT) else
swapedOrderDataVec = simdSwapedOrderDataLoad<KIND, VT, SimdWrapperType, T>(typeHolder, simdProcessor, dataVec).v;
nonEmptyMask = simdProcessor.nullEmptyCmpNe(dataVec, emptyFilterArgVec); nonEmptyMask = simdProcessor.nullEmptyCmpNe(dataVec, emptyFilterArgVec);
writeMask = nonEmptyMask;
// NULL check
nonNullMask = simdProcessor.nullEmptyCmpNe(dataVec, nullFilterArgVec);
// Exclude NULLs from the resulting set if NULL doesn't match the filters.
writeMask = isNullValueMatches ? writeMask : writeMask & nonNullMask;
nonNullOrEmptyMask = nonNullMask & nonEmptyMask;
// filters
MT prevFilterMask = initFilterMask;
// TODO name this mask literal
MT filterMask = 0xFFFF;
for (uint32_t j = 0; j < filterCount; ++j)
{
// filter using compiled filter and preloaded filter argument
if constexpr(KIND==KIND_TEXT)
filterMask = copFunctorVec[j](simdProcessor, swapedOrderDataVec, filterArgsVectors[j]);
else
filterMask = copFunctorVec[j](simdProcessor, dataVec, filterArgsVectors[j]);
filterMask = bopFunctor(prevFilterMask, filterMask); writeMask = nonEmptyMask;
prevFilterMask = filterMask; // NULL check
} nonNullMask = simdProcessor.nullEmptyCmpNe(dataVec, nullFilterArgVec);
writeMask = writeMask & filterMask; // Exclude NULLs from the resulting set if NULL doesn't match the filters.
writeMask = isNullValueMatches ? writeMask : writeMask & nonNullMask;
nonNullOrEmptyMask = nonNullMask & nonEmptyMask;
// filters
MT prevFilterMask = initFilterMask;
// TODO name this mask literal
MT filterMask = 0xFFFF;
for (uint32_t j = 0; j < filterCount; ++j)
{
// filter using compiled filter and preloaded filter argument
if constexpr(KIND==KIND_TEXT)
filterMask = copFunctorVec[j](simdProcessor, swapedOrderDataVec, filterArgsVectors[j]);
else
filterMask = copFunctorVec[j](simdProcessor, dataVec, filterArgsVectors[j]);
T* dataVecTPtr = reinterpret_cast<T*>(&dataVec); filterMask = bopFunctor(prevFilterMask, filterMask);
prevFilterMask = filterMask;
}
writeMask = writeMask & filterMask;
// vectWriteColValues iterates over the values in the source vec T* dataVecTPtr = reinterpret_cast<T*>(&dataVec);
// to store values/RIDs into dstArray/ridDstArray.
// It also sets min/max values for the block if eligible. // vectWriteColValues iterates over the values in the source vec
// !!! vectWriteColValues increases ridDstArray internally but it doesn't go // to store values/RIDs into dstArray/ridDstArray.
// outside the scope of the memory allocated to out msg. // It also sets min/max values for the block if eligible.
// vectWriteColValues is empty if outputMode == OT_RID. // !!! vectWriteColValues increases ridDstArray internally but it doesn't go
uint16_t valuesWritten = vectWriteColValues<T, VT, OUTPUT_TYPE, KIND, HAS_INPUT_RIDS>( // outside the scope of the memory allocated to out msg.
simdProcessor, writeMask, nonNullOrEmptyMask, validMinMax, ridOffset, dataVecTPtr, dstArray, min, max, // vectWriteColValues is empty if outputMode == OT_RID.
in, out, ridDstArray, ridArray); uint16_t valuesWritten = vectWriteColValues<T, VT, OUTPUT_TYPE, KIND, HAS_INPUT_RIDS>(
// Some outputType modes saves RIDs also. vectWriteRIDValues is empty for simdProcessor, writeMask, nonNullOrEmptyMask, validMinMax, ridOffset, dataVecTPtr, dstArray, min, max,
// OT_DATAVALUE, OT_BOTH(vectWriteColValues takes care about RIDs). in, out, ridDstArray, ridArray);
valuesWritten = vectWriteRIDValues<T, VT, OUTPUT_TYPE, KIND, HAS_INPUT_RIDS>( // Some outputType modes saves RIDs also. vectWriteRIDValues is empty for
simdProcessor, valuesWritten, validMinMax, ridOffset, dataVecTPtr, ridDstArray, writeMask, min, max, // OT_DATAVALUE, OT_BOTH(vectWriteColValues takes care about RIDs).
in, out, nonNullOrEmptyMask, ridArray); valuesWritten = vectWriteRIDValues<T, VT, OUTPUT_TYPE, KIND, HAS_INPUT_RIDS>(
simdProcessor, valuesWritten, validMinMax, ridOffset, dataVecTPtr, ridDstArray, writeMask, min, max,
in, out, nonNullOrEmptyMask, ridArray);
if constexpr (KIND != KIND_TEXT) if constexpr (KIND != KIND_TEXT)
vectorizedUpdateMinMax(validMinMax, nonNullOrEmptyMask, simdProcessor, dataVec, simdMin, simdMax); vectorizedUpdateMinMax(validMinMax, nonNullOrEmptyMask, simdProcessor, dataVec, simdMin, simdMax);
@ -1815,15 +1739,14 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T*
vectorizedTextUpdateMinMax(validMinMax, nonNullOrEmptyMask, simdProcessor, dataVec, simdMin, simdMax, vectorizedTextUpdateMinMax(validMinMax, nonNullOrEmptyMask, simdProcessor, dataVec, simdMin, simdMax,
swapedOrderDataVec, weightsMin, weightsMax); swapedOrderDataVec, weightsMin, weightsMax);
// Calculate bytes written // Calculate bytes written
uint16_t bytesWritten = valuesWritten * WIDTH; uint16_t bytesWritten = valuesWritten * WIDTH;
totalValuesWritten += valuesWritten; totalValuesWritten += valuesWritten;
ridDstArray += valuesWritten; ridDstArray += valuesWritten;
dstArray += bytesWritten; dstArray += bytesWritten;
rid += VECTOR_SIZE; rid += VECTOR_SIZE;
srcArray += VECTOR_SIZE; srcArray += VECTOR_SIZE;
ridArray += VECTOR_SIZE; ridArray += VECTOR_SIZE;
}
} }
if constexpr (KIND != KIND_TEXT) if constexpr (KIND != KIND_TEXT)
@ -1847,7 +1770,33 @@ void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T*
scalarFiltering<T, FT, ST, KIND>(in, out, columnFilterMode, filterSet, filterCount, filterCOPs, scalarFiltering<T, FT, ST, KIND>(in, out, columnFilterMode, filterSet, filterCount, filterCOPs,
filterValues, filterRFs, in->colType, origSrcArray, srcSize, origRidArray, filterValues, filterRFs, in->colType, origSrcArray, srcSize, origRidArray,
ridSize, processedSoFar, outputType, validMinMax, emptyValue, nullValue, ridSize, processedSoFar, outputType, validMinMax, emptyValue, nullValue,
min, max, isNullValueMatches, hasAuxCol, blockAux, emptyValueAux); min, max, isNullValueMatches, blockAux, emptyValueAux);
}
template<typename T, typename VT, bool HAS_INPUT_RIDS, int OUTPUT_TYPE,
ENUM_KIND KIND, typename FT, typename ST>
void vectorizedFiltering(NewColRequestHeader* in, ColResultHeader* out, const T* srcArray,
const uint32_t srcSize, primitives::RIDType* ridArray, const uint16_t ridSize,
ParsedColumnFilter* parsedColumnFilter, const bool validMinMax, const T emptyValue,
const T nullValue, T min, T max, const bool isNullValueMatches,
const uint8_t* blockAux, uint8_t emptyValueAux)
{
if (in->hasAuxCol)
{
vectorizedFiltering_<T, VT, HAS_INPUT_RIDS, OUTPUT_TYPE, KIND, FT, ST, true>(
in, out, srcArray, srcSize, ridArray, ridSize,
parsedColumnFilter, validMinMax, emptyValue,
nullValue, min, max, isNullValueMatches,
blockAux, emptyValueAux);
}
else
{
vectorizedFiltering_<T, VT, HAS_INPUT_RIDS, OUTPUT_TYPE, KIND, FT, ST, false>(
in, out, srcArray, srcSize, ridArray, ridSize,
parsedColumnFilter, validMinMax, emptyValue,
nullValue, min, max, isNullValueMatches,
blockAux, emptyValueAux);
}
} }
// This routine dispatches template function calls to reduce branching. // This routine dispatches template function calls to reduce branching.
@ -1858,7 +1807,7 @@ void vectorizedFilteringDispatcher(NewColRequestHeader* in, ColResultHeader* out
const bool validMinMax, const STORAGE_TYPE emptyValue, const bool validMinMax, const STORAGE_TYPE emptyValue,
const STORAGE_TYPE nullValue, STORAGE_TYPE Min, STORAGE_TYPE Max, const STORAGE_TYPE nullValue, STORAGE_TYPE Min, STORAGE_TYPE Max,
const bool isNullValueMatches, const bool isNullValueMatches,
const bool hasAuxCol, const uint8_t* blockAux, uint8_t emptyValueAux) const uint8_t* blockAux, uint8_t emptyValueAux)
{ {
// Using struct to dispatch SIMD type based on integral type T. // Using struct to dispatch SIMD type based on integral type T.
using SimdType = typename simd::IntegralToSIMD<STORAGE_TYPE, KIND>::type; using SimdType = typename simd::IntegralToSIMD<STORAGE_TYPE, KIND>::type;
@ -1873,22 +1822,22 @@ void vectorizedFilteringDispatcher(NewColRequestHeader* in, ColResultHeader* out
case OT_RID: case OT_RID:
vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_RID, KIND, FT, ST>( vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_RID, KIND, FT, ST>(
in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue,
nullValue, Min, Max, isNullValueMatches, hasAuxCol, blockAux, emptyValueAux); nullValue, Min, Max, isNullValueMatches, blockAux, emptyValueAux);
break; break;
case OT_BOTH: case OT_BOTH:
vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_BOTH, KIND, FT, ST>( vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_BOTH, KIND, FT, ST>(
in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue,
nullValue, Min, Max, isNullValueMatches, hasAuxCol, blockAux, emptyValueAux); nullValue, Min, Max, isNullValueMatches, blockAux, emptyValueAux);
break; break;
case OT_TOKEN: case OT_TOKEN:
vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_TOKEN, KIND, FT, ST>( vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_TOKEN, KIND, FT, ST>(
in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue,
nullValue, Min, Max, isNullValueMatches, hasAuxCol, blockAux, emptyValueAux); nullValue, Min, Max, isNullValueMatches, blockAux, emptyValueAux);
break; break;
case OT_DATAVALUE: case OT_DATAVALUE:
vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_DATAVALUE, KIND, FT, ST>( vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_DATAVALUE, KIND, FT, ST>(
in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue,
nullValue, Min, Max, isNullValueMatches, hasAuxCol, blockAux, emptyValueAux); nullValue, Min, Max, isNullValueMatches, blockAux, emptyValueAux);
break; break;
} }
} }
@ -1900,22 +1849,22 @@ void vectorizedFilteringDispatcher(NewColRequestHeader* in, ColResultHeader* out
case OT_RID: case OT_RID:
vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_RID, KIND, FT, ST>( vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_RID, KIND, FT, ST>(
in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue,
nullValue, Min, Max, isNullValueMatches, hasAuxCol, blockAux, emptyValueAux); nullValue, Min, Max, isNullValueMatches, blockAux, emptyValueAux);
break; break;
case OT_BOTH: case OT_BOTH:
vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_BOTH, KIND, FT, ST>( vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_BOTH, KIND, FT, ST>(
in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue,
nullValue, Min, Max, isNullValueMatches, hasAuxCol, blockAux, emptyValueAux); nullValue, Min, Max, isNullValueMatches, blockAux, emptyValueAux);
break; break;
case OT_TOKEN: case OT_TOKEN:
vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_TOKEN, KIND, FT, ST>( vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_TOKEN, KIND, FT, ST>(
in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue,
nullValue, Min, Max, isNullValueMatches, hasAuxCol, blockAux, emptyValueAux); nullValue, Min, Max, isNullValueMatches, blockAux, emptyValueAux);
break; break;
case OT_DATAVALUE: case OT_DATAVALUE:
vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_DATAVALUE, KIND, FT, ST>( vectorizedFiltering<STORAGE_TYPE, VT, hasInput, OT_DATAVALUE, KIND, FT, ST>(
in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue, in, out, srcArray, srcSize, ridArray, ridSize, parsedColumnFilter, validMinMax, emptyValue,
nullValue, Min, Max, isNullValueMatches, hasAuxCol, blockAux, emptyValueAux); nullValue, Min, Max, isNullValueMatches, blockAux, emptyValueAux);
break; break;
} }
} }
@ -1933,7 +1882,7 @@ void filterColumnData(NewColRequestHeader* in, ColResultHeader* out, uint16_t* r
const uint16_t ridSize, // Number of values in ridArray const uint16_t ridSize, // Number of values in ridArray
int* srcArray16, const uint32_t srcSize, int* srcArray16, const uint32_t srcSize,
boost::shared_ptr<ParsedColumnFilter> parsedColumnFilter, boost::shared_ptr<ParsedColumnFilter> parsedColumnFilter,
bool hasAuxCol, int* blockAux) int* blockAux)
{ {
using FT = typename IntegralTypeToFilterType<T>::type; using FT = typename IntegralTypeToFilterType<T>::type;
using ST = typename IntegralTypeToFilterSetType<T>::type; using ST = typename IntegralTypeToFilterSetType<T>::type;
@ -1996,7 +1945,7 @@ void filterColumnData(NewColRequestHeader* in, ColResultHeader* out, uint16_t* r
vectorizedFilteringDispatcher<T, KIND, FT, ST>(in, out, srcArray, srcSize, ridArray, ridSize, vectorizedFilteringDispatcher<T, KIND, FT, ST>(in, out, srcArray, srcSize, ridArray, ridSize,
parsedColumnFilter.get(), validMinMax, emptyValue, parsedColumnFilter.get(), validMinMax, emptyValue,
nullValue, Min, Max, isNullValueMatches, nullValue, Min, Max, isNullValueMatches,
hasAuxCol, reinterpret_cast<const uint8_t*>(blockAux), reinterpret_cast<const uint8_t*>(blockAux),
emptyValueAux); emptyValueAux);
return; return;
} }
@ -2006,7 +1955,7 @@ void filterColumnData(NewColRequestHeader* in, ColResultHeader* out, uint16_t* r
scalarFiltering<T, FT, ST, KIND>(in, out, columnFilterMode, filterSet, filterCount, filterCOPs, scalarFiltering<T, FT, ST, KIND>(in, out, columnFilterMode, filterSet, filterCount, filterCOPs,
filterValues, filterRFs, in->colType, srcArray, srcSize, ridArray, ridSize, filterValues, filterRFs, in->colType, srcArray, srcSize, ridArray, ridSize,
initialRID, outputType, validMinMax, emptyValue, nullValue, Min, Max, initialRID, outputType, validMinMax, emptyValue, nullValue, Min, Max,
isNullValueMatches, hasAuxCol, reinterpret_cast<const uint8_t*>(blockAux), isNullValueMatches, reinterpret_cast<const uint8_t*>(blockAux),
emptyValueAux); emptyValueAux);
} // end of filterColumnData } // end of filterColumnData
@ -2043,8 +1992,7 @@ template <typename T,
typename std::enable_if<sizeof(T) == sizeof(int32_t), T>::type* = nullptr> typename std::enable_if<sizeof(T) == sizeof(int32_t), T>::type* = nullptr>
#endif #endif
void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in, void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in,
ColResultHeader* out, ColResultHeader* out)
bool hasAuxCol)
{ {
constexpr int W = sizeof(T); constexpr int W = sizeof(T);
auto dataType = (execplan::CalpontSystemCatalog::ColDataType)in->colType.DataType; auto dataType = (execplan::CalpontSystemCatalog::ColDataType)in->colType.DataType;
@ -2053,10 +2001,10 @@ void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in,
const uint16_t ridSize = in->NVALS; const uint16_t ridSize = in->NVALS;
uint16_t* ridArray = in->getRIDArrayPtr(W); uint16_t* ridArray = in->getRIDArrayPtr(W);
const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE : BLOCK_SIZE / W; const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE : BLOCK_SIZE / W;
filterColumnData<T, KIND_FLOAT>(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter, hasAuxCol, blockAux); filterColumnData<T, KIND_FLOAT>(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter, blockAux);
return; return;
} }
_scanAndFilterTypeDispatcher<T>(in, out, hasAuxCol); _scanAndFilterTypeDispatcher<T>(in, out);
} }
template <typename T, template <typename T,
@ -2070,8 +2018,7 @@ template <typename T,
typename std::enable_if<sizeof(T) == sizeof(int64_t), T>::type* = nullptr> typename std::enable_if<sizeof(T) == sizeof(int64_t), T>::type* = nullptr>
#endif #endif
void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in, void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in,
ColResultHeader* out, ColResultHeader* out)
bool hasAuxCol)
{ {
constexpr int W = sizeof(T); constexpr int W = sizeof(T);
auto dataType = (execplan::CalpontSystemCatalog::ColDataType)in->colType.DataType; auto dataType = (execplan::CalpontSystemCatalog::ColDataType)in->colType.DataType;
@ -2080,10 +2027,10 @@ void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in,
const uint16_t ridSize = in->NVALS; const uint16_t ridSize = in->NVALS;
uint16_t* ridArray = in->getRIDArrayPtr(W); uint16_t* ridArray = in->getRIDArrayPtr(W);
const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE : BLOCK_SIZE / W; const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE : BLOCK_SIZE / W;
filterColumnData<T, KIND_FLOAT>(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter, hasAuxCol, blockAux); filterColumnData<T, KIND_FLOAT>(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter, blockAux);
return; return;
} }
_scanAndFilterTypeDispatcher<T>(in, out, hasAuxCol); _scanAndFilterTypeDispatcher<T>(in, out);
} }
template <typename T, typename std::enable_if<sizeof(T) == sizeof(int8_t) || sizeof(T) == sizeof(int16_t) || template <typename T, typename std::enable_if<sizeof(T) == sizeof(int8_t) || sizeof(T) == sizeof(int16_t) ||
@ -2100,10 +2047,9 @@ template <typename T, typename std::enable_if<sizeof(T) == sizeof(int8_t) || siz
T>::type* = nullptr> T>::type* = nullptr>
#endif #endif
void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in, void PrimitiveProcessor::scanAndFilterTypeDispatcher(NewColRequestHeader* in,
ColResultHeader* out, ColResultHeader* out)
bool hasAuxCol)
{ {
_scanAndFilterTypeDispatcher<T>(in, out, hasAuxCol); _scanAndFilterTypeDispatcher<T>(in, out);
} }
template <typename T, template <typename T,
@ -2117,15 +2063,14 @@ template <typename T,
typename std::enable_if<sizeof(T) == sizeof(int128_t), T>::type* = nullptr> typename std::enable_if<sizeof(T) == sizeof(int128_t), T>::type* = nullptr>
#endif #endif
void PrimitiveProcessor::_scanAndFilterTypeDispatcher(NewColRequestHeader* in, void PrimitiveProcessor::_scanAndFilterTypeDispatcher(NewColRequestHeader* in,
ColResultHeader* out, ColResultHeader* out)
bool hasAuxCol)
{ {
constexpr int W = sizeof(T); constexpr int W = sizeof(T);
const uint16_t ridSize = in->NVALS; const uint16_t ridSize = in->NVALS;
uint16_t* ridArray = in->getRIDArrayPtr(W); uint16_t* ridArray = in->getRIDArrayPtr(W);
const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE : BLOCK_SIZE / W; const uint32_t itemsPerBlock = logicalBlockMode ? BLOCK_SIZE : BLOCK_SIZE / W;
filterColumnData<T, KIND_DEFAULT>(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter, hasAuxCol, blockAux); filterColumnData<T, KIND_DEFAULT>(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter, blockAux);
} }
template <typename T, template <typename T,
@ -2139,8 +2084,7 @@ template <typename T,
typename std::enable_if<sizeof(T) <= sizeof(int64_t), T>::type* = nullptr> typename std::enable_if<sizeof(T) <= sizeof(int64_t), T>::type* = nullptr>
#endif #endif
void PrimitiveProcessor::_scanAndFilterTypeDispatcher(NewColRequestHeader* in, void PrimitiveProcessor::_scanAndFilterTypeDispatcher(NewColRequestHeader* in,
ColResultHeader* out, ColResultHeader* out)
bool hasAuxCol)
{ {
constexpr int W = sizeof(T); constexpr int W = sizeof(T);
using UT = typename std::conditional<std::is_unsigned<T>::value || datatypes::is_uint128_t<T>::value, T, using UT = typename std::conditional<std::is_unsigned<T>::value || datatypes::is_uint128_t<T>::value, T,
@ -2155,23 +2099,22 @@ void PrimitiveProcessor::_scanAndFilterTypeDispatcher(NewColRequestHeader* in,
dataType == execplan::CalpontSystemCatalog::TEXT) && dataType == execplan::CalpontSystemCatalog::TEXT) &&
!isDictTokenScan(in)) !isDictTokenScan(in))
{ {
filterColumnData<UT, KIND_TEXT>(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter, hasAuxCol, blockAux); filterColumnData<UT, KIND_TEXT>(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter, blockAux);
return; return;
} }
if (datatypes::isUnsigned(dataType)) if (datatypes::isUnsigned(dataType))
{ {
filterColumnData<UT, KIND_UNSIGNED>(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter, hasAuxCol, blockAux); filterColumnData<UT, KIND_UNSIGNED>(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter, blockAux);
return; return;
} }
filterColumnData<T, KIND_DEFAULT>(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter, hasAuxCol, blockAux); filterColumnData<T, KIND_DEFAULT>(in, out, ridArray, ridSize, block, itemsPerBlock, parsedColumnFilter, blockAux);
} }
// The entrypoint for block scanning and filtering. // The entrypoint for block scanning and filtering.
// The block is in in msg, out msg is used to store values|RIDs matched. // The block is in in msg, out msg is used to store values|RIDs matched.
template <typename T> template <typename T>
void PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader* in, ColResultHeader* out, void PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader* in, ColResultHeader* out)
bool hasAuxCol)
{ {
#ifdef PRIM_DEBUG #ifdef PRIM_DEBUG
auto markEvent = [&](char eventChar) auto markEvent = [&](char eventChar)
@ -2210,26 +2153,21 @@ void PrimitiveProcessor::columnScanAndFilter(NewColRequestHeader* in, ColResultH
// Sort ridArray (the row index array) if there are RIDs with this in msg // Sort ridArray (the row index array) if there are RIDs with this in msg
in->sortRIDArrayIfNeeded(W); in->sortRIDArrayIfNeeded(W);
scanAndFilterTypeDispatcher<T>(in, out, hasAuxCol); scanAndFilterTypeDispatcher<T>(in, out);
#ifdef PRIM_DEBUG #ifdef PRIM_DEBUG
markEvent('C'); markEvent('C');
#endif #endif
} }
template void primitives::PrimitiveProcessor::columnScanAndFilter<int8_t>(NewColRequestHeader*, template void primitives::PrimitiveProcessor::columnScanAndFilter<int8_t>(NewColRequestHeader*,
ColResultHeader*, ColResultHeader*);
bool);
template void primitives::PrimitiveProcessor::columnScanAndFilter<int16_t>(NewColRequestHeader*, template void primitives::PrimitiveProcessor::columnScanAndFilter<int16_t>(NewColRequestHeader*,
ColResultHeader*, ColResultHeader*);
bool);
template void primitives::PrimitiveProcessor::columnScanAndFilter<int32_t>(NewColRequestHeader*, template void primitives::PrimitiveProcessor::columnScanAndFilter<int32_t>(NewColRequestHeader*,
ColResultHeader*, ColResultHeader*);
bool);
template void primitives::PrimitiveProcessor::columnScanAndFilter<int64_t>(NewColRequestHeader*, template void primitives::PrimitiveProcessor::columnScanAndFilter<int64_t>(NewColRequestHeader*,
ColResultHeader*, ColResultHeader*);
bool);
template void primitives::PrimitiveProcessor::columnScanAndFilter<int128_t>(NewColRequestHeader*, template void primitives::PrimitiveProcessor::columnScanAndFilter<int128_t>(NewColRequestHeader*,
ColResultHeader*, ColResultHeader*);
bool);
} // namespace primitives } // namespace primitives

12
primitives/linux-port/primitiveprocessor.h
View File

@ -396,20 +396,20 @@ class PrimitiveProcessor
template <typename T, typename std::enable_if<sizeof(T) == sizeof(int8_t) || sizeof(T) == sizeof(int16_t) || template <typename T, typename std::enable_if<sizeof(T) == sizeof(int8_t) || sizeof(T) == sizeof(int16_t) ||
sizeof(T) == sizeof(int128_t), sizeof(T) == sizeof(int128_t),
T>::type* = nullptr> T>::type* = nullptr>
void scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out, bool hasAuxCol); void scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out);
template <typename T, typename std::enable_if<sizeof(T) == sizeof(int32_t), T>::type* = nullptr> template <typename T, typename std::enable_if<sizeof(T) == sizeof(int32_t), T>::type* = nullptr>
void scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out, bool hasAuxCol); void scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out);
template <typename T, typename std::enable_if<sizeof(T) == sizeof(int64_t), T>::type* = nullptr> template <typename T, typename std::enable_if<sizeof(T) == sizeof(int64_t), T>::type* = nullptr>
void scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out, bool hasAuxCol); void scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out);
template <typename T, typename std::enable_if<sizeof(T) <= sizeof(int64_t), T>::type* = nullptr> template <typename T, typename std::enable_if<sizeof(T) <= sizeof(int64_t), T>::type* = nullptr>
void _scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out, bool hasAuxCol); void _scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out);
template <typename T, typename std::enable_if<sizeof(T) == sizeof(int128_t), T>::type* = nullptr> template <typename T, typename std::enable_if<sizeof(T) == sizeof(int128_t), T>::type* = nullptr>
void _scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out, bool hasAuxCol); void _scanAndFilterTypeDispatcher(NewColRequestHeader* in, ColResultHeader* out);
template <typename T> template <typename T>
void columnScanAndFilter(NewColRequestHeader* in, ColResultHeader* out, bool hasAuxCol); void columnScanAndFilter(NewColRequestHeader* in, ColResultHeader* out);
boost::shared_ptr<ParsedColumnFilter> parseColumnFilter(const uint8_t* filterString, uint32_t colWidth, boost::shared_ptr<ParsedColumnFilter> parseColumnFilter(const uint8_t* filterString, uint32_t colWidth,
uint32_t colType, uint32_t filterCount, uint32_t colType, uint32_t filterCount,

21
primitives/primproc/columncommand.cpp
View File

@ -203,7 +203,7 @@ void ColumnCommand::_loadData()
bpp->physIO += blocksRead; bpp->physIO += blocksRead;
bpp->touchedBlocks += blocksToLoad; bpp->touchedBlocks += blocksToLoad;
if (_hasAuxCol) if (hasAuxCol_)
{ {
BRM::LBID_t* lbidsAux = (BRM::LBID_t*)alloca(1 * sizeof(BRM::LBID_t)); BRM::LBID_t* lbidsAux = (BRM::LBID_t*)alloca(1 * sizeof(BRM::LBID_t));
uint8_t** blockPtrsAux = (uint8_t**)alloca(1 * sizeof(uint8_t*)); uint8_t** blockPtrsAux = (uint8_t**)alloca(1 * sizeof(uint8_t*));
@ -288,9 +288,10 @@ template <int W>
void ColumnCommand::_issuePrimitive() void ColumnCommand::_issuePrimitive()
{ {
using IntegralType = typename datatypes::WidthToSIntegralType<W>::type; using IntegralType = typename datatypes::WidthToSIntegralType<W>::type;
primMsg->hasAuxCol = hasAuxCol_;
// Down the call stack the code presumes outMsg buffer has enough space to store // Down the call stack the code presumes outMsg buffer has enough space to store
// ColRequestHeader + uint16_t Rids[8192] + IntegralType[8192]. // ColRequestHeader + uint16_t Rids[8192] + IntegralType[8192].
bpp->getPrimitiveProcessor().columnScanAndFilter<IntegralType>(primMsg, outMsg, _hasAuxCol); bpp->getPrimitiveProcessor().columnScanAndFilter<IntegralType>(primMsg, outMsg);
} // _issuePrimitive() } // _issuePrimitive()
void ColumnCommand::updateCPDataNarrow() void ColumnCommand::updateCPDataNarrow()
@ -541,7 +542,7 @@ void ColumnCommand::createCommand(ByteStream& bs)
bs >> BOP; bs >> BOP;
bs >> filterCount; bs >> filterCount;
bs >> tmp8; bs >> tmp8;
_hasAuxCol = tmp8; hasAuxCol_ = tmp8;
deserializeInlineVector(bs, lastLbid); deserializeInlineVector(bs, lastLbid);
Command::createCommand(bs); Command::createCommand(bs);
@ -575,7 +576,7 @@ void ColumnCommand::createCommand(execplan::CalpontSystemCatalog::ColType& aColT
bs >> BOP; bs >> BOP;
bs >> filterCount; bs >> filterCount;
bs >> tmp8; bs >> tmp8;
_hasAuxCol = tmp8; hasAuxCol_ = tmp8;
deserializeInlineVector(bs, lastLbid); deserializeInlineVector(bs, lastLbid);
Command::createCommand(bs); Command::createCommand(bs);
@ -585,7 +586,7 @@ void ColumnCommand::resetCommand(ByteStream& bs)
{ {
bs >> lbid; bs >> lbid;
if (_hasAuxCol) if (hasAuxCol_)
bs >> lbidAux; bs >> lbidAux;
} }
@ -848,14 +849,14 @@ void ColumnCommand::nextLBID()
{ {
lbid += colType.colWidth; lbid += colType.colWidth;
if (_hasAuxCol) if (hasAuxCol_)
lbidAux += execplan::AUX_COL_WIDTH; lbidAux += execplan::AUX_COL_WIDTH;
} }
void ColumnCommand::duplicate(ColumnCommand* cc) void ColumnCommand::duplicate(ColumnCommand* cc)
{ {
cc->_isScan = _isScan; cc->_isScan = _isScan;
cc->_hasAuxCol = _hasAuxCol; cc->hasAuxCol_ = hasAuxCol_;
cc->traceFlags = traceFlags; cc->traceFlags = traceFlags;
cc->filterString = filterString; cc->filterString = filterString;
cc->colType.colDataType = colType.colDataType; cc->colType.colDataType = colType.colDataType;
@ -887,7 +888,7 @@ bool ColumnCommand::operator==(const ColumnCommand& cc) const
if (_isScan != cc._isScan) if (_isScan != cc._isScan)
return false; return false;
if (_hasAuxCol != cc._hasAuxCol) if (hasAuxCol_ != cc.hasAuxCol_)
return false; return false;
if (BOP != cc.BOP) if (BOP != cc.BOP)
@ -919,7 +920,7 @@ bool ColumnCommand::operator!=(const ColumnCommand& cc) const
ColumnCommand& ColumnCommand::operator=(const ColumnCommand& c) ColumnCommand& ColumnCommand::operator=(const ColumnCommand& c)
{ {
_isScan = c._isScan; _isScan = c._isScan;
_hasAuxCol = c._hasAuxCol; hasAuxCol_ = c.hasAuxCol_;
traceFlags = c.traceFlags; traceFlags = c.traceFlags;
filterString = c.filterString; filterString = c.filterString;
colType.colDataType = c.colType.colDataType; colType.colDataType = c.colType.colDataType;
@ -958,7 +959,7 @@ void ColumnCommand::getLBIDList(uint32_t loopCount, vector<int64_t>* lbids)
for (i = firstLBID; i <= lastLBID; i++) for (i = firstLBID; i <= lastLBID; i++)
lbids->push_back(i); lbids->push_back(i);
if (_hasAuxCol) if (hasAuxCol_)
{ {
firstLBID = lbidAux; firstLBID = lbidAux;
lastLBID = firstLBID + (loopCount * execplan::AUX_COL_WIDTH) - 1; lastLBID = firstLBID + (loopCount * execplan::AUX_COL_WIDTH) - 1;

4
primitives/primproc/columncommand.h
View File

@ -79,7 +79,7 @@ class ColumnCommand : public Command
bool hasAuxCol() const bool hasAuxCol() const
{ {
return _hasAuxCol; return hasAuxCol_;
} }
uint64_t getLBIDAux() const uint64_t getLBIDAux() const
{ {
@ -173,7 +173,7 @@ class ColumnCommand : public Command
uint32_t baseMsgLength; uint32_t baseMsgLength;
uint64_t lbid; uint64_t lbid;
bool _hasAuxCol; bool hasAuxCol_;
uint64_t lbidAux; uint64_t lbidAux;
uint32_t traceFlags; // probably move this to Command uint32_t traceFlags; // probably move this to Command
uint8_t BOP; uint8_t BOP;

52
tests/primitives_column_scan_and_filter.cpp
View File

@ -136,7 +136,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan1Byte)
in->NVALS = 0; in->NVALS = 0;
pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
EXPECT_EQ(out->NVALS, 8160); EXPECT_EQ(out->NVALS, 8160);
@ -160,7 +160,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan1ByteVectorized)
in->NVALS = 0; in->NVALS = 0;
pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
EXPECT_EQ(out->NVALS, 8160); EXPECT_EQ(out->NVALS, 8160);
@ -196,7 +196,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan2Bytes)
in->NVALS = 0; in->NVALS = 0;
pp.setBlockPtr((int*)readBlockFromLiteralArray("col2block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col2block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
EXPECT_EQ(out->NVALS, 4096); EXPECT_EQ(out->NVALS, 4096);
@ -223,7 +223,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan4Bytes)
in->NVALS = 0; in->NVALS = 0;
pp.setBlockPtr((int*)readBlockFromLiteralArray("col4block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col4block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
EXPECT_EQ(out->NVALS, 2048); EXPECT_EQ(out->NVALS, 2048);
@ -251,7 +251,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes)
in->NVALS = 0; in->NVALS = 0;
pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
ASSERT_EQ(out->NVALS, 1024); ASSERT_EQ(out->NVALS, 1024);
@ -287,7 +287,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan2Bytes1EqFilter)
&input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]);
pp.setBlockPtr((int*)readBlockFromLiteralArray("col2block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col2block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
ASSERT_EQ(out->NVALS, 51); ASSERT_EQ(out->NVALS, 51);
@ -315,7 +315,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan1ByteUsingRID)
rids[1] = 17; rids[1] = 17;
pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
ASSERT_EQ(out->NVALS, 2); ASSERT_EQ(out->NVALS, 2);
@ -344,7 +344,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan1ByteUsingMultipleRIDs)
rids[126] = 8189; rids[126] = 8189;
pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
ASSERT_EQ(out->NVALS, expectedNVALS); ASSERT_EQ(out->NVALS, expectedNVALS);
@ -376,7 +376,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan4Bytes1EqFilter)
&input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]);
pp.setBlockPtr((int*)readBlockFromLiteralArray("col4block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col4block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
ASSERT_EQ(out->NVALS, 8); ASSERT_EQ(out->NVALS, 8);
@ -404,7 +404,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan4BytesUsingMultipleRIDs)
rids[126] = 1020; rids[126] = 1020;
pp.setBlockPtr((int*)readBlockFromLiteralArray("col4block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col4block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
@ -438,7 +438,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan4Bytes2Filters)
memcpy(args->val, &tmp, in->colType.DataSize); memcpy(args->val, &tmp, in->colType.DataSize);
pp.setBlockPtr((int*)readBlockFromLiteralArray("col4block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col4block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
ASSERT_EQ(out->NVALS, 9); ASSERT_EQ(out->NVALS, 9);
@ -477,7 +477,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes1EqFilter)
&input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]); &input[sizeof(NewColRequestHeader) + sizeof(ColArgs) + in->colType.DataSize]);
pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
ASSERT_EQ(out->NVALS, 11); ASSERT_EQ(out->NVALS, 11);
@ -511,7 +511,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan8BytesUsingMultipleRIDs)
rids[126] = 1020; rids[126] = 1020;
pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
@ -547,7 +547,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2CompFilters)
memcpy(args->val, &tmp, in->colType.DataSize); memcpy(args->val, &tmp, in->colType.DataSize);
pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
ASSERT_EQ(out->NVALS, 33); ASSERT_EQ(out->NVALS, 33);
@ -587,7 +587,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFilters)
memcpy(args->val, &tmp, in->colType.DataSize); memcpy(args->val, &tmp, in->colType.DataSize);
pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
@ -631,7 +631,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFiltersRID)
rids[1] = 100; rids[1] = 100;
pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
ASSERT_EQ(out->NVALS, 1); ASSERT_EQ(out->NVALS, 1);
@ -662,7 +662,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2FiltersRIDOutputRid)
memcpy(args->val, &tmp, in->colType.DataSize); memcpy(args->val, &tmp, in->colType.DataSize);
pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<RIDType>(getFirstRIDArrayPosition(out), 0); results = getValuesArrayPosition<RIDType>(getFirstRIDArrayPosition(out), 0);
ASSERT_EQ(out->NVALS, 33); ASSERT_EQ(out->NVALS, 33);
@ -701,7 +701,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan8Bytes2EqFiltersRIDOutputBoth)
memcpy(args->val, &tmp, in->colType.DataSize); memcpy(args->val, &tmp, in->colType.DataSize);
pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col8block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
ASSERT_EQ(out->NVALS, 33); ASSERT_EQ(out->NVALS, 33);
@ -742,7 +742,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan1Byte2CompFilters)
args->val[0] = '4'; args->val[0] = '4';
pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col1block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<UT>(getFirstValueArrayPosition(out), 0);
ASSERT_EQ(out->NVALS, 32); ASSERT_EQ(out->NVALS, 32);
@ -791,7 +791,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan4Bytes2CompFiltersOutputRID)
memcpy(&args->val[in->colType.DataSize], &ridTmp, 2); memcpy(&args->val[in->colType.DataSize], &ridTmp, 2);
pp.setBlockPtr((int*) readBlockFromLiteralArray("col4block.cdf", block)); pp.setBlockPtr((int*) readBlockFromLiteralArray("col4block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = reinterpret_cast<int16_t*>(&output[sizeof(ColResultHeader)]); results = reinterpret_cast<int16_t*>(&output[sizeof(ColResultHeader)]);
ASSERT_EQ(out->NVALS, 2); ASSERT_EQ(out->NVALS, 2);
@ -823,7 +823,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan8BytesDouble2CompFilters)
memcpy(args->val, &tmp, sizeof(tmp)); memcpy(args->val, &tmp, sizeof(tmp));
pp.setBlockPtr((int*)readBlockFromLiteralArray("col_double_block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col_double_block.cdf", block));
pp.columnScanAndFilter<int64_t>(in, out, false); pp.columnScanAndFilter<int64_t>(in, out);
results = getValuesArrayPosition<IntegralType>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<IntegralType>(getFirstValueArrayPosition(out), 0);
ASSERT_EQ(out->NVALS, 8); ASSERT_EQ(out->NVALS, 8);
@ -857,7 +857,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan4BytesFloat2CompFiltersOutputBoth)
memcpy(args->val, &tmp, sizeof(tmp)); memcpy(args->val, &tmp, sizeof(tmp));
pp.setBlockPtr((int*)readBlockFromLiteralArray("col_float_block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col_float_block.cdf", block));
pp.columnScanAndFilter<int32_t>(in, out, false); pp.columnScanAndFilter<int32_t>(in, out);
ASSERT_EQ(out->NVALS, 8); ASSERT_EQ(out->NVALS, 8);
@ -892,7 +892,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan4BytesNegFloat2CompFiltersOutputBoth)
memcpy(args->val, &tmp, sizeof(tmp)); memcpy(args->val, &tmp, sizeof(tmp));
pp.setBlockPtr((int*)readBlockFromLiteralArray("col_neg_float.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col_neg_float.cdf", block));
pp.columnScanAndFilter<int32_t>(in, out, false); pp.columnScanAndFilter<int32_t>(in, out);
ASSERT_EQ(out->NVALS, 19); ASSERT_EQ(out->NVALS, 19);
for (i = 0; i < out->NVALS; i++) for (i = 0; i < out->NVALS; i++)
@ -926,7 +926,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan8BytesNegDouble2CompFilters)
memcpy(args->val, &tmp, sizeof(tmp)); memcpy(args->val, &tmp, sizeof(tmp));
pp.setBlockPtr((int*)readBlockFromLiteralArray("col_neg_double.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col_neg_double.cdf", block));
pp.columnScanAndFilter<int64_t>(in, out, false); pp.columnScanAndFilter<int64_t>(in, out);
ASSERT_EQ(out->NVALS, 19); ASSERT_EQ(out->NVALS, 19);
@ -950,7 +950,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan16Bytes)
in->NVALS = 0; in->NVALS = 0;
pp.setBlockPtr((int*)readBlockFromLiteralArray("col16block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col16block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<IntegralType>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<IntegralType>(getFirstValueArrayPosition(out), 0);
@ -992,7 +992,7 @@ TEST_F(ColumnScanFilterTest, ColumnScan16Bytes2CompFilters)
memcpy(args->val, &tmp, in->colType.DataSize); memcpy(args->val, &tmp, in->colType.DataSize);
pp.setBlockPtr((int*)readBlockFromLiteralArray("col16block.cdf", block)); pp.setBlockPtr((int*)readBlockFromLiteralArray("col16block.cdf", block));
pp.columnScanAndFilter<IntegralType>(in, out, false); pp.columnScanAndFilter<IntegralType>(in, out);
results = getValuesArrayPosition<IntegralType>(getFirstValueArrayPosition(out), 0); results = getValuesArrayPosition<IntegralType>(getFirstValueArrayPosition(out), 0);

3
versioning/BRM/dbrm.cpp
View File

@ -43,6 +43,7 @@
#include "configcpp.h" #include "configcpp.h"
#include "sessionmanagerserver.h" #include "sessionmanagerserver.h"
#include "messagequeuepool.h" #include "messagequeuepool.h"
#include "blocksize.h"
#define DBRM_DLLEXPORT #define DBRM_DLLEXPORT
#include "dbrm.h" #include "dbrm.h"
#undef DBRM_DLLEXPORT #undef DBRM_DLLEXPORT
@ -4513,7 +4514,7 @@ void DBRM::addToLBIDList(uint32_t sessionID, vector<LBID_t>& lbidList)
} }
} }
uint32_t extentNumAux = fbo / (execplan::AUX_COL_WIDTH * 1024); uint32_t extentNumAux = fbo / ((getExtentRows() * execplan::AUX_COL_WIDTH) / BLOCK_SIZE);
for (auto iter = extentMap[tableOid].begin(); iter != extentMap[tableOid].end(); iter++) for (auto iter = extentMap[tableOid].begin(); iter != extentMap[tableOid].end(); iter++)
{ {

20
writeengine/bulk/we_bulkload.cpp
View File

@ -265,7 +265,7 @@ int BulkLoad::loadJobInfo(const string& fullName, bool bUseTempJobFile, int argc
return rc; return rc;
} }
Job& curJob = const_cast<Job&>(fJobInfo.getJob()); Job& curJob = fJobInfo.getJob();
string logFile, errlogFile; string logFile, errlogFile;
logFile = std::string(MCSLOGDIR) + "/cpimport/" + "Job_" + Convertor::int2Str(curJob.id) + LOG_SUFFIX; logFile = std::string(MCSLOGDIR) + "/cpimport/" + "Job_" + Convertor::int2Str(curJob.id) + LOG_SUFFIX;
errlogFile = errlogFile =
@ -319,6 +319,8 @@ int BulkLoad::loadJobInfo(const string& fullName, bool bUseTempJobFile, int argc
execplan::CalpontSystemCatalog::OID tableAUXColOid; execplan::CalpontSystemCatalog::OID tableAUXColOid;
std::string tblName; std::string tblName;
std::string curTblName = curJob.jobTableList[i].tblName; std::string curTblName = curJob.jobTableList[i].tblName;
// Parse out <tablename> from [<schemaname>.]<tablename> string
string::size_type startName = curTblName.rfind('.'); string::size_type startName = curTblName.rfind('.');
if (startName == std::string::npos) if (startName == std::string::npos)
@ -376,18 +378,10 @@ int BulkLoad::loadJobInfo(const string& fullName, bool bUseTempJobFile, int argc
// tableAUXColOid = 0 // tableAUXColOid = 0
if (tableAUXColOid > 3000) if (tableAUXColOid > 3000)
{ {
JobColumn curColumn; JobColumn curColumn("aux", tableAUXColOid, execplan::AUX_COL_DATATYPE_STRING,
curColumn.colName = "aux"; execplan::AUX_COL_WIDTH, execplan::AUX_COL_WIDTH,
curColumn.mapOid = tableAUXColOid; execplan::AUX_COL_COMPRESSION_TYPE, execplan::AUX_COL_COMPRESSION_TYPE,
curColumn.typeName = execplan::AUX_COL_DATATYPE_STRING; execplan::AUX_COL_MINVALUE, execplan::AUX_COL_MAXVALUE, true, 1);
curColumn.width = execplan::AUX_COL_WIDTH;
curColumn.definedWidth = execplan::AUX_COL_WIDTH;
curColumn.compressionType = execplan::AUX_COL_COMPRESSION_TYPE;
curColumn.dctnry.fCompressionType = execplan::AUX_COL_COMPRESSION_TYPE;
curColumn.fMinIntSat = execplan::AUX_COL_MINVALUE;
curColumn.fMaxIntSat = execplan::AUX_COL_MAXVALUE;
curColumn.fWithDefault = true;
curColumn.fDefaultUInt = 1;
curJob.jobTableList[i].colList.push_back(curColumn); curJob.jobTableList[i].colList.push_back(curColumn);
JobFieldRef fieldRef(BULK_FLDCOL_COLUMN_DEFAULT, curJob.jobTableList[i].colList.size() - 1); JobFieldRef fieldRef(BULK_FLDCOL_COLUMN_DEFAULT, curJob.jobTableList[i].colList.size() - 1);
curJob.jobTableList[i].fFldRefs.push_back(fieldRef); curJob.jobTableList[i].fFldRefs.push_back(fieldRef);

62
writeengine/server/we_dmlcommandproc.cpp
View File

@ -91,6 +91,27 @@ WE_DMLCommandProc::~WE_DMLCommandProc()
dbRootExtTrackerVec.clear(); dbRootExtTrackerVec.clear();
} }
void WE_DMLCommandProc::processAuxCol(const std::vector<std::string>& origVals,
WriteEngine::ColValueList& colValuesList,
WriteEngine::DictStrList& dicStringList)
{
WriteEngine::ColTupleList auxColTuples;
WriteEngine::dictStr auxDicStrings;
for (uint32_t j = 0; j < origVals.size(); j++)
{
WriteEngine::ColTuple auxColTuple;
auxColTuple.data = (uint8_t)1;
auxColTuples.push_back(auxColTuple);
//@Bug 2515. Only pass string values to write engine
auxDicStrings.push_back("");
}
colValuesList.push_back(auxColTuples);
//@Bug 2515. Only pass string values to write engine
dicStringList.push_back(auxDicStrings);
}
uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std::string& err) uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std::string& err)
{ {
uint8_t rc = 0; uint8_t rc = 0;
@ -443,22 +464,7 @@ uint8_t WE_DMLCommandProc::processSingleInsert(messageqcpp::ByteStream& bs, std:
// MCOL-5021 // MCOL-5021
if ((i == numcols - 1) && (tableAUXColOid > 3000)) if ((i == numcols - 1) && (tableAUXColOid > 3000))
{ {
WriteEngine::ColTupleList auxColTuples; processAuxCol(origVals, colValuesList, dicStringList);
WriteEngine::dictStr auxDicStrings;
for (uint32_t j = 0; j < origVals.size(); j++)
{
WriteEngine::ColTuple auxColTuple;
auxColTuple.data = (uint8_t)1;
auxColTuples.push_back(auxColTuple);
//@Bug 2515. Only pass string values to write engine
auxDicStrings.push_back("");
}
colValuesList.push_back(auxColTuples);
//@Bug 2515. Only pass string values to write engine
dicStringList.push_back(auxDicStrings);
} }
++row_iterator; ++row_iterator;
@ -1385,22 +1391,7 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
// MCOL-5021 // MCOL-5021
if ((i == numcols - 1) && (tableAUXColOid > 3000)) if ((i == numcols - 1) && (tableAUXColOid > 3000))
{ {
WriteEngine::ColTupleList auxColTuples; processAuxCol(origVals, colValuesList, dicStringList);
WriteEngine::dictStr auxDicStrings;
for (uint32_t j = 0; j < origVals.size(); j++)
{
WriteEngine::ColTuple auxColTuple;
auxColTuple.data = (uint8_t)1;
auxColTuples.push_back(auxColTuple);
//@Bug 2515. Only pass string values to write engine
auxDicStrings.push_back("");
}
colValuesList.push_back(auxColTuples);
//@Bug 2515. Only pass string values to write engine
dicStringList.push_back(auxDicStrings);
} }
++row_iterator; ++row_iterator;
@ -1502,7 +1493,6 @@ uint8_t WE_DMLCommandProc::processBatchInsert(messageqcpp::ByteStream& bs, std::
return rc; return rc;
} }
#if 0
uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err, uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err,
ByteStream::quadbyte& PMId) ByteStream::quadbyte& PMId)
{ {
@ -1665,8 +1655,9 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
return rc; return rc;
} }
//@Bug 5996 validate hwm before starts // @Bug 5996 validate hwm before starts
rc = validateColumnHWMs(ridList, systemCatalogPtr, colDBRootExtentInfo, "Starting"); // TODO MCOL-5021 hasAuxCol is hardcoded to false; add support here.
rc = validateColumnHWMs(ridList, systemCatalogPtr, colDBRootExtentInfo, "Starting", false);
if (rc != 0) if (rc != 0)
{ {
@ -2233,7 +2224,6 @@ uint8_t WE_DMLCommandProc::processBatchInsertBinary(messageqcpp::ByteStream& bs,
CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000); CalpontSystemCatalog::removeCalpontSystemCatalog(sessionId | 0x80000000);
return rc; return rc;
} }
#endif
uint8_t WE_DMLCommandProc::commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err) uint8_t WE_DMLCommandProc::commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err)
{ {

8
writeengine/server/we_dmlcommandproc.h
View File

@ -78,8 +78,8 @@ class WE_DMLCommandProc
EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string& err); EXPORT uint8_t rollbackVersion(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t processBatchInsert(messageqcpp::ByteStream& bs, std::string& err, EXPORT uint8_t processBatchInsert(messageqcpp::ByteStream& bs, std::string& err,
ByteStream::quadbyte& PMId); ByteStream::quadbyte& PMId);
//EXPORT uint8_t processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err, EXPORT uint8_t processBatchInsertBinary(messageqcpp::ByteStream& bs, std::string& err,
// ByteStream::quadbyte& PMId); ByteStream::quadbyte& PMId);
EXPORT uint8_t commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err); EXPORT uint8_t commitBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t commitBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err); EXPORT uint8_t commitBatchAutoOff(messageqcpp::ByteStream& bs, std::string& err);
EXPORT uint8_t rollbackBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err); EXPORT uint8_t rollbackBatchAutoOn(messageqcpp::ByteStream& bs, std::string& err);
@ -127,6 +127,10 @@ class WE_DMLCommandProc
const std::vector<BRM::FileInfo>& files, const std::vector<BRM::FileInfo>& files,
const std::vector<BRM::OID_t>& oidsToFlush, std::string& err); const std::vector<BRM::OID_t>& oidsToFlush, std::string& err);
void processAuxCol(const std::vector<std::string>& origVals,
WriteEngine::ColValueList& colValuesList,
WriteEngine::DictStrList& dicStringList);
bool fIsFirstBatchPm; bool fIsFirstBatchPm;
std::map<uint32_t, rowgroup::RowGroup*> rowGroups; std::map<uint32_t, rowgroup::RowGroup*> rowGroups;
std::map<uint32_t, dmlpackage::UpdateDMLPackage> cpackages; std::map<uint32_t, dmlpackage::UpdateDMLPackage> cpackages;

2
writeengine/server/we_readthread.cpp
View File

@ -156,13 +156,11 @@ void DmlReadThread::operator()()
break; break;
} }
#if 0
case WE_SVR_BATCH_INSERT_BINARY: case WE_SVR_BATCH_INSERT_BINARY:
{ {
rc = fWeDMLprocessor->processBatchInsertBinary(ibs, errMsg, PMId); rc = fWeDMLprocessor->processBatchInsertBinary(ibs, errMsg, PMId);
break; break;
} }
#endif
case WE_SVR_GET_WRITTEN_LBIDS: case WE_SVR_GET_WRITTEN_LBIDS:
{ {

3
writeengine/shared/we_config.cpp
View File

@ -188,7 +188,8 @@ void Config::checkReload()
const std::string fastDeleteTemp = cf->getConfig("WriteEngine", "FastDelete"); const std::string fastDeleteTemp = cf->getConfig("WriteEngine", "FastDelete");
if (fastDeleteTemp.length() != 0 && boost::iequals(fastDeleteTemp, "true")) if (fastDeleteTemp.length() != 0 &&
(fastDeleteTemp == "y" || fastDeleteTemp == "Y"))
{ {
m_FastDelete = true; m_FastDelete = true;
} }

32
writeengine/shared/we_type.h
View File

@ -436,6 +436,38 @@ struct JobColumn /** @brief Job Column Structure */
, fDefaultWideDecimal(0) , fDefaultWideDecimal(0)
{ {
} }
JobColumn(const std::string& colName_, OID mapOid_, const std::string& typeName_,
int width_, int definedWidth_, int compressionType_, int dctnryCompressionType_,
int64_t minIntSat_, uint64_t maxIntSat_, bool withDefault_,
unsigned long long defaultUInt_)
: colName(colName_)
, mapOid(mapOid_)
, dataType(execplan::CalpontSystemCatalog::INT)
, weType(WR_INT)
, typeName(typeName_)
, emptyVal(nullptr)
, width(width_)
, definedWidth(definedWidth_)
, dctnryWidth(0)
, precision(0)
, scale(0)
, fNotNull(false)
, fFldColRelation(BULK_FLDCOL_COLUMN_FIELD)
, colType(' ')
, compressionType(compressionType_)
, autoIncFlag(false)
, fMinIntSat(minIntSat_)
, fMaxIntSat(maxIntSat_)
, fMinDblSat(0)
, fMaxDblSat(0)
, fWithDefault(withDefault_)
, fDefaultInt(0)
, fDefaultUInt(defaultUInt_)
, fDefaultDbl(0.0)
, fDefaultWideDecimal(0)
{
dctnry.fCompressionType = dctnryCompressionType_;
}
}; };
typedef std::vector<JobColumn> JobColList; /** @brief column value list */ typedef std::vector<JobColumn> JobColList; /** @brief column value list */

14
writeengine/wrapper/we_colop.cpp
View File

@ -1412,12 +1412,12 @@ int ColumnOp::openColumnFile(Column& column, std::string& segFile, bool useTmpSu
if (!isValid(column)) if (!isValid(column))
return ERR_INVALID_PARAM; return ERR_INVALID_PARAM;
std::string mode; std::string mode = "r";
if (isReadOnly) if (!isReadOnly)
mode = "r"; {
else
mode = "r+b"; mode = "r+b";
}
// open column data file // open column data file
column.dataFile.pFile = column.dataFile.pFile =
@ -1817,10 +1817,9 @@ int ColumnOp::writeRowsReadOnly(Column& curCol, uint64_t totalRow, const RIDList
uint64_t i = 0, curRowId; uint64_t i = 0, curRowId;
int dataFbo, dataBio, curDataFbo = -1; int dataFbo, dataBio, curDataFbo = -1;
unsigned char dataBuf[BYTE_PER_BLOCK]; unsigned char dataBuf[BYTE_PER_BLOCK];
bool bExit = false;
int rc = NO_ERROR; int rc = NO_ERROR;
while (!bExit) while (i < totalRow)
{ {
curRowId = ridList[i]; curRowId = ridList[i];
@ -1845,9 +1844,6 @@ int ColumnOp::writeRowsReadOnly(Column& curCol, uint64_t totalRow, const RIDList
} }
i++; i++;
if (i >= totalRow)
bExit = true;
} }
return rc; return rc;

8
writeengine/xml/we_xmljob.h
View File

@ -84,6 +84,14 @@ class XMLJob : public XMLOp
return fJob; return fJob;
} }
/**
* @brief Get reference to job structure
*/
Job& getJob()
{
return fJob;
}
/** /**
* @brief Load job information * @brief Load job information
* @param fileName Name of Job XML file to be read * @param fileName Name of Job XML file to be read